diff --git a/cmd/main.go b/cmd/main.go index e28d6fb2a5..69695781d4 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -43,6 +43,8 @@ import ( "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" @@ -607,6 +609,14 @@ admission policy installation; once an Installation exists it is the authority o os.Exit(1) } + // Build the extensions for the variant we resolved above. + extensionRegistry := enterprise.New(variant, eoptions.Options{ + MultiTenant: multiTenant, + Cloud: isCloudBuild(), + ManageCRDs: manageCRDs, + UseV3CRDs: v3CRDs, + }) + options := options.ControllerOptions{ DetectedProvider: provider, Variant: variant, @@ -621,6 +631,7 @@ admission policy installation; once an Installation exists it is the authority o ESMigration: elasticIsMigrating, UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, + Extensions: extensionRegistry, } // Before we start any controllers, make sure our options are valid. diff --git a/pkg/common/validation/apiserver/validation.go b/pkg/common/validation/apiserver/validation.go index 14df010555..71ec852b99 100644 --- a/pkg/common/validation/apiserver/validation.go +++ b/pkg/common/validation/apiserver/validation.go @@ -44,9 +44,9 @@ func validateContainerPorts(container corev1.Container) field.ErrorList { ports := container.Ports // Validate if the port name can be attributed to the container. for _, port := range ports { - if (port.Name == render.APIServerPortName && container.Name != string(render.APIServerContainerName)) || - (port.Name == render.QueryServerPortName && container.Name != string(render.TigeraAPIServerQueryServerContainerName)) || - (port.Name == render.L7AdmissionControllerPortName && container.Name != string(render.L7AdmissionControllerContainerName)) { + if (port.Name == render.APIServerPortName && container.Name != render.APIServerContainerName) || + (port.Name == render.QueryServerPortName && container.Name != render.TigeraAPIServerQueryServerContainerName) || + (port.Name == render.L7AdmissionControllerPortName && container.Name != render.L7AdmissionControllerContainerName) { msg := fmt.Sprintf("port name %s is not valid for container %s", port.Name, container.Name) allErrs = append(allErrs, field.Invalid(fldPath, port.Name, msg)) } diff --git a/pkg/common/validation/overrides_test.go b/pkg/common/validation/overrides_test.go index 454cd4dcc8..5e1ce42bba 100644 --- a/pkg/common/validation/overrides_test.go +++ b/pkg/common/validation/overrides_test.go @@ -298,7 +298,7 @@ var _ = Describe("Test overrides validation (APIServerDeployment - Container.Por It("should accept custom valid ContainerPorts", func() { overrides.Spec.Template.Spec.Containers = []opv1.APIServerDeploymentContainer{ { - Name: string(render.APIServerContainerName), + Name: render.APIServerContainerName, Ports: []opv1.APIServerDeploymentContainerPort{ { Name: render.APIServerPortName, @@ -307,7 +307,7 @@ var _ = Describe("Test overrides validation (APIServerDeployment - Container.Por }, }, { - Name: string(render.TigeraAPIServerQueryServerContainerName), + Name: render.TigeraAPIServerQueryServerContainerName, Ports: []opv1.APIServerDeploymentContainerPort{ { Name: render.QueryServerPortName, @@ -316,7 +316,7 @@ var _ = Describe("Test overrides validation (APIServerDeployment - Container.Por }, }, { - Name: string(render.L7AdmissionControllerContainerName), + Name: render.L7AdmissionControllerContainerName, Ports: []opv1.APIServerDeploymentContainerPort{ { Name: render.L7AdmissionControllerPortName, @@ -351,7 +351,7 @@ var _ = Describe("Test overrides validation (APIServerDeployment - Container.Por ), Entry("queryserver PortName specified in the wrong container", opv1.APIServerDeploymentContainer{ - Name: string(render.APIServerContainerName), + Name: render.APIServerContainerName, Ports: []opv1.APIServerDeploymentContainerPort{ { Name: render.QueryServerPortName, @@ -363,7 +363,7 @@ var _ = Describe("Test overrides validation (APIServerDeployment - Container.Por ), Entry("l7admctrl PortName specified in the wrong container", opv1.APIServerDeploymentContainer{ - Name: string(render.APIServerContainerName), + Name: render.APIServerContainerName, Ports: []opv1.APIServerDeploymentContainerPort{ { Name: render.L7AdmissionControllerPortName, diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index 10a05aab84..d93ec69a0a 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -26,7 +26,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" + ctrl "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -40,6 +40,7 @@ import ( "github.com/tigera/operator/pkg/common/validation" apiserver "github.com/tigera/operator/pkg/common/validation/apiserver" webhooksvalidation "github.com/tigera/operator/pkg/common/validation/webhooks" + "github.com/tigera/operator/pkg/controller" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration/datastoremigration" @@ -49,11 +50,10 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" - "github.com/tigera/operator/pkg/render/common/authentication" "github.com/tigera/operator/pkg/render/common/networkpolicy" - "github.com/tigera/operator/pkg/render/common/rbacmanagement" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/render/webhooks" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -73,10 +73,11 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { tierWatchReady: &utils.ReadyFlag{}, migrationWatchReady: &utils.ReadyFlag{}, opts: opts, + ext: opts.Extensions.APIServer(), } r.status.Run(opts.ShutdownContext) - c, err := ctrlruntime.NewController("apiserver-controller", mgr, controller.Options{Reconciler: r}) + c, err := ctrlruntime.NewController("apiserver-controller", mgr, ctrl.Options{Reconciler: r}) if err != nil { return fmt.Errorf("failed to create apiserver-controller: %w", err) } @@ -104,48 +105,10 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("apiserver-controller failed to watch ConfigMap %s: %w", render.K8sSvcEndpointConfigMapName, err) } - if opts.Variant.IsEnterprise() { - // Watched so a toggle re-renders the rules gated on it. - if err = utils.AddConfigMapWatch(c, rbacmanagement.ConfigMapName, common.CalicoNamespace, &handler.EnqueueRequestForObject{}); err != nil { - return fmt.Errorf("apiserver-controller failed to watch ConfigMap %s: %w", rbacmanagement.ConfigMapName, err) - } - - // Watch for changes to ApplicationLayer - err = c.WatchObject(&operatorv1.ApplicationLayer{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch ApplicationLayer resource: %v", err) - } - - // Watch for changes to primary resource ManagementCluster - err = c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch primary resource: %v", err) - } - - // Watch for changes to primary resource ManagementClusterConnection - err = c.WatchObject(&operatorv1.ManagementClusterConnection{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch primary resource: %v", err) - } - - for _, namespace := range []string{common.OperatorNamespace(), render.APIServerNamespace} { - for _, secretName := range []string{render.VoltronTunnelSecretName, render.ManagerTLSSecretName} { - if err = utils.AddSecretsWatch(c, secretName, namespace); err != nil { - return fmt.Errorf("apiserver-controller failed to watch the Secret resource: %v", err) - } - } - } - - if err = utils.AddSecretsWatch(c, render.VoltronLinseedPublicCert, common.OperatorNamespace()); err != nil { - return fmt.Errorf("apiserver-controller failed to watch the Secret resource: %v", err) - } - - // Watch for changes to authentication - err = c.WatchObject(&operatorv1.Authentication{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("apiserver-controller failed to watch resource: %w", err) - } - + // The variant extension registers the enterprise watches it needs (the management + // cluster CRs, ApplicationLayer, Authentication, and the tunnel secrets). + if err = opts.Extensions.APIServer().Watches(c); err != nil { + return fmt.Errorf("apiserver-controller failed to set up extension watches: %w", err) } // Watch for the namespace(s) managed by this controller. @@ -242,6 +205,7 @@ type ReconcileAPIServer struct { tierWatchReady *utils.ReadyFlag migrationWatchReady *utils.ReadyFlag opts options.ControllerOptions + ext extensions.APIServerExtension } // Reconcile reads that state of the cluster for a APIServer object and makes changes based on the state read @@ -345,16 +309,6 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - // Since apiserver and queryserver may have different UID:GID at run-time, we need to produce this secret in separate volumes and with different permissions. - var queryServerTLSSecretCertificateManagementOnly certificatemanagement.KeyPairInterface - if installationSpec.CertificateManagement != nil { - queryServerTLSSecretCertificateManagementOnly, err = certificateManager.GetOrCreateKeyPair(r.client, "query-server-tls", common.OperatorNamespace(), dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, r.opts.ClusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to get or create tls key pair", err, reqLogger) - return reconcile.Result{}, err - } - } - certificateManager.AddToStatusManager(r.status, render.APIServerNamespace) pullSecrets, err := utils.GetInstallationPullSecrets(installationSpec, r.client) @@ -363,121 +317,47 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re return reconcile.Result{}, err } - // Query enterprise-only data. - var trustedBundle certificatemanagement.TrustedBundle - var applicationLayer *operatorv1.ApplicationLayer - var managementCluster *operatorv1.ManagementCluster - var managementClusterConnection *operatorv1.ManagementClusterConnection - var keyValidatorConfig authentication.KeyValidatorConfig - var rbacManagementEnabled bool - includeV3NetworkPolicy := false - - if installationSpec.Variant.IsEnterprise() { - trustedBundle, err = certificateManager.CreateNamedTrustedBundleFromSecrets(render.APIServerResourceName, r.client, - common.OperatorNamespace(), false) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the trusted bundle", err, reqLogger) - return reconcile.Result{}, err - } - - rbacManagementEnabled, err = utils.RBACManagementEnabled(ctx, r.client, installationSpec.Variant, r.opts.MultiTenant) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading the RBAC management UI ConfigMap", err, reqLogger) - return reconcile.Result{}, err - } - - applicationLayer, err = utils.GetApplicationLayer(ctx, r.client) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ApplicationLayer", err, reqLogger) + // Run the variant extension: it validates the configuration and produces the + // enterprise render data (the trusted bundle, the query server certificate, the + // management cluster CRs, the OIDC config, and the resolved L7 sidecar images). For + // the core operator this is a no-op and the render inputs carries no extension data. + ci := controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: installationSpec, + ClusterDomain: r.opts.ClusterDomain, + }, + Client: r.client, + CertificateManager: certificateManager, + } + ci, extraKeyPairs, err := r.ext.ExtendInputs(ctx, ci) + if err != nil { + if reason, ok := extensions.DegradedReason(err); ok { + r.status.SetDegraded(reason, err.Error(), nil, reqLogger) + if reason == operatorv1.ResourceNotReady { + // The controller watches what the extension is waiting on, so let the + // watch trigger the next reconcile. + return reconcile.Result{}, nil + } return reconcile.Result{}, err } + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing the API server extension", err, reqLogger) + return reconcile.Result{}, err + } + trustedBundle := ci.RenderInputs.TrustedBundle + // The webhooks component (v3-CRD mode) needs the ManagementCluster to register the + // managed-cluster webhook. Reading it requires the enterprise CRDs. + var managementCluster *operatorv1.ManagementCluster + if r.opts.Variant.IsEnterprise() { managementCluster, err = utils.GetManagementCluster(ctx, r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) return reconcile.Result{}, err } - - managementClusterConnection, err = utils.GetManagementClusterConnection(ctx, r.client) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementClusterConnection", err, reqLogger) - return reconcile.Result{}, err - } - - if managementClusterConnection != nil && managementCluster != nil { - err = fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported") - r.status.SetDegraded(operatorv1.ResourceValidationError, "", err, reqLogger) - return reconcile.Result{}, err - } - - // Management cluster only: check if the tunnel CA secret has been created. The apiserver mounts this secret so - // it can sign certificates for managed clusters. If the managementCluster has not been defaulted then we should - // not degrade. This is because the manager_controller exits the reconcile loop if the apiserver is not available. - if managementCluster != nil && managementCluster.Spec.TLS != nil && !r.opts.MultiTenant { - tunnelSecretName := managementCluster.Spec.TLS.SecretName - // The manager_controller should have written this secret. We know this since spec.TLS has been defaulted. - // If the secret does not exist, we degrade this controller. - _, err := utils.GetSecret(ctx, r.client, tunnelSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to fetch the tunnel secret", err, reqLogger) - return reconcile.Result{}, err - } - } - - prometheusCertificate, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate", err, reqLogger) - return reconcile.Result{}, err - } else if prometheusCertificate != nil { - trustedBundle.AddCertificates(prometheusCertificate) - } - - if managementClusterConnection != nil { - voltronLinseedCert, err := certificateManager.GetCertificate(r.client, render.VoltronLinseedPublicCert, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Failed to retrieve %s", render.VoltronLinseedPublicCert), err, reqLogger) - return reconcile.Result{}, err - } - if voltronLinseedCert != nil { - trustedBundle.AddCertificates(voltronLinseedCert) - } - } - - var authenticationCR *operatorv1.Authentication - // Fetch the Authentication spec. If present, we use it to configure user authentication. - authenticationCR, err = utils.GetAuthentication(ctx, r.client) - if err != nil && !errors.IsNotFound(err) { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error while fetching Authentication", err, reqLogger) - return reconcile.Result{}, err - } - - if authenticationCR != nil && authenticationCR.Status.State == operatorv1.TigeraStatusReady { - if utils.DexEnabled(authenticationCR) { - // Do not include DEX TLS Secret Name if authentication CR does not have type Dex - secret := render.DexTLSSecretName - certificate, err := certificateManager.GetCertificate(r.client, secret, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve %s", secret), - err, reqLogger) - return reconcile.Result{}, err - } else if certificate == nil { - reqLogger.Info(fmt.Sprintf("Waiting for secret '%s' to become available", secret)) - r.status.SetDegraded(operatorv1.ResourceNotReady, - fmt.Sprintf("Waiting for secret '%s' to become available", secret), - nil, reqLogger) - return reconcile.Result{}, nil - } - trustedBundle.AddCertificates(certificate) - } - - keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain, false) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get KeyValidator Config", err, reqLogger) - return reconcile.Result{}, err - } - } } + includeV3NetworkPolicy := false + // Ensure the calico-system tier exists, before rendering any network policies within it. // // The creation of the Tier depends on this controller to reconcile it's non-NetworkPolicy resources so that @@ -508,20 +388,15 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re } // Create a component handler to manage the rendered component. - handler := utils.NewComponentHandler(log, r.client, r.scheme, instance) - - // Determine the tenant namespaces whose calico-apiserver ServiceAccount should be granted Linseed access. - // For zero/single-tenant clusters this is empty (the calico-system API server is covered by its own - // ClusterRoleBinding); for multi-tenant management clusters it is every tenant namespace, so each tenant's - // calico-apiserver identity is authorized against Linseed. - var bindingNamespaces []string - if r.opts.MultiTenant { - bindingNamespaces, err = utils.TenantNamespaces(ctx, r.client, nil) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading tenant namespaces", err, reqLogger) - return reconcile.Result{}, err - } - } + handler := utils.NewComponentHandler( + log, + r.client, + r.scheme, + instance, + utils.WithModifier(func(c render.Component) render.Component { + return r.ext.Modify(c, ci.RenderInputs) + }), + ) var holdCutover bool if !r.opts.UseV3CRDs { @@ -541,23 +416,15 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re Installation: installationSpec, APIServer: &instance.Spec, ForceHostNetwork: false, - ApplicationLayer: applicationLayer, - ManagementCluster: managementCluster, - ManagementClusterConnection: managementClusterConnection, TLSKeyPair: tlsSecret, PullSecrets: pullSecrets, OpenShift: r.opts.DetectedProvider.IsOpenShift(), TrustedBundle: trustedBundle, MultiTenant: r.opts.MultiTenant, - BindingNamespaces: bindingNamespaces, - KeyValidatorConfig: keyValidatorConfig, KubernetesVersion: r.opts.KubernetesVersion, ClusterDomain: r.opts.ClusterDomain, - Cloud: r.opts.Cloud, RequiresAggregationServer: !r.opts.UseV3CRDs, HoldAPIServiceCutover: holdCutover, - RBACManagementEnabled: rbacManagementEnabled, - QueryServerTLSKeyPairCertificateManagementOnly: queryServerTLSSecretCertificateManagementOnly, } var components []render.Component @@ -566,6 +433,9 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re certKeyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(tlsSecret, true, true), } + for _, kp := range extraKeyPairs { + certKeyPairOptions = append(certKeyPairOptions, rcertificatemanagement.NewKeyPairOption(kp, true, true)) + } if r.opts.UseV3CRDs { // If using v3 CRDs, we render the webhooks component that handles various RBAC and validation // responsibilities. The ordering of resources here is important to avoid a deadlock: @@ -653,10 +523,14 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re } // Check BYO certificate expiry warnings. - certificatemanagement.CheckKeyPairWarnings(map[string]certificatemanagement.KeyPairInterface{ + keyPairWarnings := map[string]certificatemanagement.KeyPairInterface{ render.CalicoAPIServerTLSSecretName: tlsSecret, webhooks.WebhooksTLSSecretName: webhooksTLS, - }, r.status) + } + for _, kp := range extraKeyPairs { + keyPairWarnings[kp.GetName()] = kp + } + certificatemanagement.CheckKeyPairWarnings(keyPairWarnings, r.status) if holdCutover { r.status.SetDegraded(operatorv1.ResourceNotReady, "Waiting for the Calico API server to become ready before repointing the projectcalico.org/v3 APIService", nil, reqLogger) diff --git a/pkg/controller/apiserver/apiserver_controller_test.go b/pkg/controller/apiserver/apiserver_controller_test.go index 5e4d10f54b..c0a97c5e27 100644 --- a/pkg/controller/apiserver/apiserver_controller_test.go +++ b/pkg/controller/apiserver/apiserver_controller_test.go @@ -34,7 +34,6 @@ import ( "k8s.io/apimachinery/pkg/types" apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/reconcile" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" @@ -51,7 +50,6 @@ import ( "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" - "github.com/tigera/operator/pkg/render/common/rbacmanagement" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/tls" "github.com/tigera/operator/test" @@ -167,12 +165,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Create(ctx, installation)).To(BeNil()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -226,12 +226,14 @@ var _ = Describe("apiserver controller tests", func() { })).ToNot(HaveOccurred()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -279,12 +281,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Create(ctx, apiSecret)).ShouldNot(HaveOccurred()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, ClusterDomain: dns.DefaultClusterDomain, @@ -304,12 +308,14 @@ var _ = Describe("apiserver controller tests", func() { secretName := "calico-apiserver-certs" r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -322,16 +328,49 @@ var _ = Describe("apiserver controller tests", func() { Expect(secret.GetOwnerReferences()).To(HaveLen(1)) }) + It("should wait for the Dex TLS secret rather than fail the reconcile", func() { + Expect(cli.Create(ctx, installation)).To(BeNil()) + + // Status is dropped on create, so mark Authentication ready explicitly. + auth := &operatorv1.Authentication{} + Expect(cli.Get(ctx, client.ObjectKey{Name: "tigera-secure"}, auth)).NotTo(HaveOccurred()) + auth.Status.State = "Ready" + Expect(cli.Status().Update(ctx, auth)).NotTo(HaveOccurred()) + + Expect(cli.Delete(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: render.DexTLSSecretName, Namespace: common.OperatorNamespace()}, + })).NotTo(HaveOccurred()) + + r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), + client: cli, + scheme: scheme, + status: mockStatus, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + opts: options.ControllerOptions{ + Extensions: testExtensions, + Variant: operatorv1.CalicoEnterprise, + DetectedProvider: operatorv1.ProviderNone, + }, + } + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + mockStatus.AssertCalled(GinkgoT(), "SetDegraded", operatorv1.ResourceNotReady, mock.Anything, mock.Anything, mock.Anything) + }) + It("should render calico-system policy when tier and tier watch are ready", func() { Expect(cli.Create(ctx, installation)).To(BeNil()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -350,12 +389,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -372,12 +413,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Create(ctx, installation)).To(BeNil()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: notReady, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -397,12 +440,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Create(ctx, installation)).To(BeNil()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -424,12 +469,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -449,12 +496,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Create(ctx, installation)).To(BeNil()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: notReady, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -475,12 +524,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.Calico, DetectedProvider: operatorv1.ProviderNone, }, @@ -494,143 +545,6 @@ var _ = Describe("apiserver controller tests", func() { }) }) - // These cover the controller's half: reading the ConfigMap and handing the value to - // the renderer. - Context("RBAC management UI feature gate", func() { - // gatedRule is where the gate's value is observable in the rendered output. - gatedRule := rbacv1.PolicyRule{ - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterrolebindings", "rolebindings"}, - Verbs: []string{"get", "list", "watch", "create", "update", "delete"}, - } - - writeGate := func(value string) { - Expect(cli.Create(ctx, &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: rbacmanagement.ConfigMapName, - Namespace: common.CalicoNamespace, - }, - Data: map[string]string{rbacmanagement.ConfigMapKey: value}, - })).NotTo(HaveOccurred()) - } - - // reconcileAPIServer reconciles at the given tenancy. - reconcileAPIServer := func(multiTenant bool) { - r := ReconcileAPIServer{ - client: cli, - scheme: scheme, - status: mockStatus, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - opts: options.ControllerOptions{ - Variant: operatorv1.CalicoEnterprise, - DetectedProvider: operatorv1.ProviderNone, - MultiTenant: multiTenant, - }, - } - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - } - - // networkAdminRules reconciles a single-tenant cluster -- the only tenancy that - // renders tigera-network-admin -- and returns its rules. - networkAdminRules := func() []rbacv1.PolicyRule { - reconcileAPIServer(false) - - cr := rbacv1.ClusterRole{} - Expect(cli.Get(ctx, client.ObjectKey{Name: "tigera-network-admin"}, &cr)).NotTo(HaveOccurred()) - return cr.Rules - } - - BeforeEach(func() { - Expect(cli.Create(ctx, installation)).NotTo(HaveOccurred()) - }) - - It("withholds the rules when the admin has not created the ConfigMap", func() { - Expect(networkAdminRules()).NotTo(ContainElement(gatedRule)) - }) - - It("adds the rules once the admin enables the feature", func() { - writeGate("true") - Expect(networkAdminRules()).To(ContainElement(gatedRule)) - }) - - It("withholds the rules when the admin sets the value to false", func() { - writeGate("false") - Expect(networkAdminRules()).NotTo(ContainElement(gatedRule)) - }) - - // The gated rules ride on tigera-network-admin, which a multi-tenant cluster - // never renders -- which is why the gate needs no tenancy term of its own. - It("does not render tigera-network-admin at all on a multi-tenant management cluster", func() { - writeGate("true") - reconcileAPIServer(true) - - err := cli.Get(ctx, client.ObjectKey{Name: "tigera-network-admin"}, &rbacv1.ClusterRole{}) - Expect(kerror.IsNotFound(err)).To(BeTrue(), "expected no tigera-network-admin ClusterRole under multi-tenancy") - }) - - // A managed cluster carries tigera-network-admin too, so the read must not be - // skipped there. - It("reads the gate on a managed cluster", func() { - Expect(cli.Create(ctx, &operatorv1.ManagementClusterConnection{ - ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, - })).NotTo(HaveOccurred()) - writeGate("true") - - Expect(networkAdminRules()).To(ContainElement(gatedRule)) - }) - - // An unreadable ConfigMap is unknown state, not absent, so it degrades rather - // than rendering as disabled. - It("degrades and requeues when the ConfigMap cannot be read", func() { - readErr := fmt.Errorf("the API server is having a bad day") - failing := ctrlrfake.DefaultFakeClientBuilder(scheme). - WithInterceptorFuncs(interceptor.Funcs{ - Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { - if _, ok := obj.(*corev1.ConfigMap); ok && key.Name == rbacmanagement.ConfigMapName { - return readErr - } - return c.Get(ctx, key, obj, opts...) - }, - }).Build() - - // Re-plant what the reconcile needs on the new client. - certificateManager, err := certificatemanager.Create(failing, nil, "cluster.local", common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - Expect(failing.Create(ctx, certificateManager.KeyPair().Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) - Expect(failing.Create(ctx, &operatorv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}})).NotTo(HaveOccurred()) - Expect(failing.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - // The shared installation carries a resourceVersion that Create would reject. - freshInstallation := installation.DeepCopy() - freshInstallation.ResourceVersion = "" - Expect(failing.Create(ctx, freshInstallation)).NotTo(HaveOccurred()) - - degraded := &status.MockStatus{} - degraded.On("OnCRFound").Return() - degraded.On("SetMetaData", mock.Anything).Return() - degraded.On("AddCertificateSigningRequests", mock.Anything).Return().Maybe() - degraded.On("RemoveCertificateSigningRequests", mock.Anything).Return().Maybe() - degraded.On("SetDegraded", operatorv1.ResourceReadError, - "Error reading the RBAC management UI ConfigMap", readErr.Error(), mock.Anything).Return().Once() - - r := ReconcileAPIServer{ - client: failing, - scheme: scheme, - status: degraded, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - opts: options.ControllerOptions{ - Variant: operatorv1.CalicoEnterprise, - DetectedProvider: operatorv1.ProviderNone, - }, - } - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).To(MatchError(readErr)) - degraded.AssertExpectations(GinkgoT()) - }) - }) - Context("Reconcile for Condition status", func() { generation := int64(2) BeforeEach(func() { @@ -654,12 +568,14 @@ var _ = Describe("apiserver controller tests", func() { } Expect(cli.Create(ctx, ts)).NotTo(HaveOccurred()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -686,12 +602,14 @@ var _ = Describe("apiserver controller tests", func() { Status: operatorv1.TigeraStatusStatus{}, } r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -738,12 +656,14 @@ var _ = Describe("apiserver controller tests", func() { } Expect(cli.Create(ctx, ts)).NotTo(HaveOccurred()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -807,12 +727,14 @@ var _ = Describe("apiserver controller tests", func() { } Expect(cli.Create(ctx, ts)).NotTo(HaveOccurred()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -911,12 +833,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(err).NotTo(HaveOccurred()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -940,12 +864,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(err).NotTo(HaveOccurred()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, @@ -970,12 +896,14 @@ var _ = Describe("apiserver controller tests", func() { It("Should reconcile multi-cluster setup for a management cluster for a multiple tenant", func() { r := ReconcileAPIServer{ + ext: multiTenantExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: multiTenantExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, MultiTenant: true, @@ -1021,12 +949,14 @@ var _ = Describe("apiserver controller tests", func() { })).NotTo(HaveOccurred()) r := ReconcileAPIServer{ + ext: multiTenantExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: multiTenantExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, MultiTenant: true, @@ -1040,7 +970,7 @@ var _ = Describe("apiserver controller tests", func() { // per tenant namespace. crb := rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: render.APIServerLinseedAccessClusterRoleName}, + ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-linseed-access"}, } Expect(test.GetResource(cli, &crb)).To(BeNil()) Expect(crb.Subjects).To(ConsistOf( @@ -1056,12 +986,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Create(ctx, installation)).To(BeNil()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: true, @@ -1100,12 +1032,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Create(ctx, ossInstallation)).To(BeNil()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.Calico, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: true, @@ -1128,12 +1062,14 @@ var _ = Describe("apiserver controller tests", func() { Expect(cli.Create(ctx, installation)).To(BeNil()) r := ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, UseV3CRDs: false, @@ -1185,12 +1121,14 @@ var _ = Describe("apiserver controller tests", func() { })).NotTo(HaveOccurred()) r = ReconcileAPIServer{ + ext: testExtensions.APIServer(), client: cli, scheme: scheme, status: mockStatus, tierWatchReady: ready, migrationWatchReady: &utils.ReadyFlag{}, opts: options.ControllerOptions{ + Extensions: testExtensions, Variant: operatorv1.CalicoEnterprise, DetectedProvider: operatorv1.ProviderNone, }, diff --git a/pkg/controller/apiserver/apiserver_suite_test.go b/pkg/controller/apiserver/apiserver_suite_test.go index ae0a254ba0..3465df4f16 100644 --- a/pkg/controller/apiserver/apiserver_suite_test.go +++ b/pkg/controller/apiserver/apiserver_suite_test.go @@ -24,8 +24,19 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" ) +// testExtensions is the registry the API server controller tests reconcile with, so +// the componentHandler applies the API server modifier. +var testExtensions = enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{}) + +// multiTenantExtensions is the same registry in multi-tenant mode. +var multiTenantExtensions = enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{MultiTenant: true}) + func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter), zap.UseDevMode(true), zap.Level(uzap.NewAtomicLevelAt(uzap.DebugLevel)))) gomega.RegisterFailHandler(ginkgo.Fail) diff --git a/pkg/controller/clusterconnection/clusterconnection_controller.go b/pkg/controller/clusterconnection/clusterconnection_controller.go index 4c55406bd9..c71b6a98b9 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller.go @@ -16,7 +16,6 @@ package clusterconnection import ( "context" - "errors" "fmt" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" @@ -32,7 +31,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" + ctrl "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -42,6 +41,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" @@ -49,6 +49,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/goldmane" @@ -75,15 +76,15 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { reconciler := newReconciler(mgr.GetClient(), mgr.GetScheme(), statusManager, opts.DetectedProvider, tierWatchReady, clusterInfoWatchReady, opts) // Create a new controller - c, err := ctrlruntime.NewController(controllerName, mgr, controller.Options{Reconciler: reconciler}) + c, err := ctrlruntime.NewController(controllerName, mgr, ctrl.Options{Reconciler: reconciler}) if err != nil { return fmt.Errorf("failed to create %s: %w", controllerName, err) } - if opts.Variant.IsEnterprise() { - // Watch for changes to License and Tier, as their status is used as input to determine whether network policy should be reconciled by this controller. - go utils.WaitToAddLicenseKeyWatch(c, opts.K8sClientset, log, nil) + if err = opts.Extensions.ClusterConnection().Watches(c, opts.K8sClientset); err != nil { + return fmt.Errorf("%s failed to add variant watches: %w", controllerName, err) } + go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, tierWatchReady) go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, []types.NamespacedName{ @@ -131,30 +132,6 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("clusterconnection-controller failed to watch management-cluster-connection Tigerastatus: %w", err) } - if opts.Variant.IsEnterprise() { - err = c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("%s failed to watch primary resource: %w", controllerName, err) - } - - // Watch for changes to the secrets associated with the PacketCapture APIs. - if err = utils.AddSecretsWatch(c, render.PacketCaptureServerCert, common.OperatorNamespace()); err != nil { - return fmt.Errorf("%s failed to watch Secret resource %s: %w", controllerName, render.PacketCaptureServerCert, err) - } - // Watch for changes to the secrets associated with Prometheus. - if err = utils.AddSecretsWatch(c, monitor.PrometheusServerTLSSecretName, common.OperatorNamespace()); err != nil { - return fmt.Errorf("%s failed to watch Secret resource %s: %w", controllerName, monitor.PrometheusServerTLSSecretName, err) - } - - if err = utils.AddSecretsWatch(c, certificatemanagement.CASecretName, common.OperatorNamespace()); err != nil { - return fmt.Errorf("%s failed to watch Secret resource %s: %w", controllerName, certificatemanagement.CASecretName, err) - } - - if err = imageset.AddImageSetWatch(c); err != nil { - return fmt.Errorf("%s failed to watch ImageSet: %w", controllerName, err) - } - } - return nil } @@ -173,10 +150,10 @@ func newReconciler( scheme: schema, provider: p, status: statusMgr, - clusterDomain: opts.ClusterDomain, - variant: opts.Variant, tierWatchReady: tierWatchReady, clusterInfoWatchReady: clusterInfoWatchReady, + opts: opts, + ext: opts.Extensions.ClusterConnection(), } c.status.Run(opts.ShutdownContext) return c @@ -191,12 +168,12 @@ type ReconcileConnection struct { scheme *runtime.Scheme provider operatorv1.Provider status status.StatusManager - clusterDomain string - variant operatorv1.ProductVariant tierWatchReady *utils.ReadyFlag clusterInfoWatchReady *utils.ReadyFlag resolvedPodProxies []*httpproxy.Config lastAvailabilityTransition metav1.Time + opts options.ControllerOptions + ext extensions.ClusterConnectionExtension } // Reconcile reads that state of the cluster for a ManagementClusterConnection object and makes changes based on the @@ -246,34 +223,18 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R } } - // Verify the cluster doesn't also have the ManagementCluster CRD installed. - if r.variant.IsEnterprise() { - managementCluster, err := utils.GetManagementCluster(ctx, r.cli) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) - return reconcile.Result{}, err - } - - if managementCluster != nil { - err = fmt.Errorf("having both a ManagementCluster and a ManagementClusterConnection is not supported") - r.status.SetDegraded(operatorv1.ResourceValidationError, "", err, reqLogger) - return reconcile.Result{}, err - } - } - // Validate that the cluster information watch is ready. if !r.clusterInfoWatchReady.IsReady() { r.status.SetDegraded(operatorv1.ResourceNotReady, "Waiting for clusterInfoWatchReady watch to be established", err, reqLogger) return reconcile.Result{RequeueAfter: utils.StandardRetry}, nil } - if err = validate(managementClusterConnection, installationSpec.Variant); err != nil { - r.status.SetDegraded(operatorv1.ResourceValidationError, "ManagementClusterConnection.Spec.Impersonation must be unset when Installation.Spec.Variant = Calico", err, reqLogger) + preDefaultPatchFrom := client.MergeFrom(managementClusterConnection.DeepCopy()) + if err = r.ext.ValidateAndDefault(managementClusterConnection); err != nil { + r.status.SetDegraded(operatorv1.ResourceValidationError, "Invalid ManagementClusterConnection configuration", err, reqLogger) return reconcile.Result{}, err } - - preDefaultPatchFrom := client.MergeFrom(managementClusterConnection.DeepCopy()) - fillDefaults(managementClusterConnection, installationSpec.Variant) + fillDefaults(managementClusterConnection) if err = r.cli.Patch(ctx, managementClusterConnection, preDefaultPatchFrom); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, err.Error(), err, reqLogger) } @@ -285,20 +246,37 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R log.V(2).Info("Loaded ManagementClusterConnection config", "config", managementClusterConnection) - certificateManager, err := certificatemanager.Create(r.cli, installationSpec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + certificateManager, err := certificatemanager.Create(r.cli, installationSpec, r.opts.ClusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err } - includeSystem := false - if managementClusterConnection.Spec.TLS.CA == operatorv1.CATypePublic { - if r.variant == operatorv1.Calico { - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "Guardian CA cannot be public in Calico.", nil, reqLogger) - return reconcile.Result{}, nil + // Run the variant extension: it validates the configuration (a cluster cannot be + // both a management and a managed cluster) and produces the Enterprise-specific + // Guardian inputs the controller reads back below (the managed cluster version and + // the license-gated egress policy flag). For the core operator this is a no-op and + // the render inputs carries no extension data, so the OSS defaults apply. + ci := controller.Inputs{ + RenderInputs: render.Inputs{Installation: installationSpec, ClusterDomain: r.opts.ClusterDomain}, + Client: r.cli, + CertificateManager: certificateManager, + } + ci, _, err = r.ext.ExtendInputs(ctx, ci) + if err != nil { + if reason, ok := extensions.DegradedReason(err); ok { + r.status.SetDegraded(reason, err.Error(), nil, reqLogger) + if reason == operatorv1.ResourceNotReady { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err } - includeSystem = true + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing the clusterconnection extension", err, reqLogger) + return reconcile.Result{}, err } + guardianData, haveGuardianData := render.GuardianRenderDataFromInputs(ci.RenderInputs) + + includeSystem := managementClusterConnection.Spec.TLS.CA == operatorv1.CATypePublic trustedBundle, err := certificateManager.CreateNamedTrustedBundleFromSecrets(render.GuardianDeploymentName, r.cli, common.OperatorNamespace(), includeSystem, @@ -307,9 +285,12 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the trusted bundle", err, reqLogger) } + // In the OSS (Whisker) path Guardian connects with its own client keypair. The + // Enterprise path uses the tunnel secret instead, so when the extension supplied + // its Guardian inputs we skip creating this keypair. var guardianKeyPair certificatemanagement.KeyPairInterface - if !r.variant.IsEnterprise() { - guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.clusterDomain) + if !haveGuardianData { + guardianCertificateNames := dns.GetServiceDNSNames("guardian", render.GuardianNamespace, r.opts.ClusterDomain) guardianCertificateNames = append(guardianCertificateNames, "localhost", "127.0.0.1") guardianKeyPair, err = certificateManager.GetOrCreateKeyPair(r.cli, render.GuardianKeyPairSecret, whisker.WhiskerNamespace, guardianCertificateNames) if err != nil { @@ -411,8 +392,8 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying clusterInformation", err, reqLogger) return reconcile.Result{}, err } - if r.variant.IsEnterprise() { - managedClusterVersion = clusterInformation.Spec.CNXVersion + if haveGuardianData { + managedClusterVersion = guardianData.Version } else { managedClusterVersion = clusterInformation.Spec.CalicoVersion } @@ -423,18 +404,9 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{RequeueAfter: utils.StandardRetry}, nil } - var includeEgressNetworkPolicy bool - if r.variant.IsEnterprise() { - // Ensure the license can support enterprise policy, before rendering any network policies within it. - if license, err := utils.FetchLicenseKey(ctx, r.cli); err == nil { - if utils.IsFeatureActive(license, common.EgressAccessControlFeature) { - includeEgressNetworkPolicy = true - } - } else if !k8serrors.IsNotFound(err) { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying license", err, reqLogger) - return reconcile.Result{}, err - } - } + // The Enterprise extension gates the domain-based egress rules on the license; the + // OSS default is to leave them disabled. + includeEgressNetworkPolicy := guardianData.IncludeEgressNetworkPolicy // Ensure the calico-system tier exists, before rendering any network policies within it. var tierAvailable bool @@ -445,7 +417,15 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, err } - ch := utils.NewComponentHandler(log, r.cli, r.scheme, managementClusterConnection) + ch := utils.NewComponentHandler( + log, + r.cli, + r.scheme, + managementClusterConnection, + utils.WithModifier(func(c render.Component) render.Component { + return r.ext.Modify(c, ci.RenderInputs) + }), + ) guardianCfg := &render.GuardianConfiguration{ URL: managementClusterConnection.Spec.ManagementClusterAddr, PodProxies: r.resolvedPodProxies, @@ -485,7 +465,7 @@ func (r *ReconcileConnection) Reconcile(ctx context.Context, request reconcile.R } } - if err = imageset.ApplyImageSet(ctx, r.cli, r.variant, components...); err != nil { + if err = imageset.ApplyImageSet(ctx, r.cli, r.opts.Variant, components...); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error with images from ImageSet", err, reqLogger) return reconcile.Result{}, err } @@ -510,25 +490,11 @@ func (r *ReconcileConnection) maintainFinalizer(ctx context.Context, managementC return utils.MaintainInstallationFinalizer(ctx, r.cli, managementClusterConnection, render.GuardianFinalizer, &guardianDeployment) } -func validate(cr *operatorv1.ManagementClusterConnection, variant operatorv1.ProductVariant) error { - if variant == operatorv1.Calico && cr.Spec.Impersonation != nil { - return errors.New("ManagementClusterConnection.Spec.Impersonation must be unset when Installation.Spec.Variant = Calico") - } - return nil -} - -func fillDefaults(cr *operatorv1.ManagementClusterConnection, variant operatorv1.ProductVariant) { +func fillDefaults(cr *operatorv1.ManagementClusterConnection) { if cr.Spec.TLS == nil { cr.Spec.TLS = &operatorv1.ManagementClusterTLS{} } if cr.Spec.TLS.CA == "" { cr.Spec.TLS.CA = operatorv1.CATypeTigera } - if variant.IsEnterprise() && cr.Spec.Impersonation == nil { - cr.Spec.Impersonation = &operatorv1.Impersonation{ - Users: []string{}, - Groups: []string{}, - ServiceAccounts: []string{}, - } - } } diff --git a/pkg/controller/clusterconnection/clusterconnection_controller_enterprise_test.go b/pkg/controller/clusterconnection/clusterconnection_controller_enterprise_test.go new file mode 100644 index 0000000000..0e272ab8ad --- /dev/null +++ b/pkg/controller/clusterconnection/clusterconnection_controller_enterprise_test.go @@ -0,0 +1,512 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/stretchr/testify/mock" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/clusterconnection" + "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/test" +) + +// These tests cover the Calico Enterprise behavior of the ManagementClusterConnection +// controller: the enterprise images, the license/tier-gated calico-system network +// policy, and impersonation. The shared (variant-agnostic) controller mechanics live +// in clusterconnection_controller_test.go. +var _ = Describe("ManagementClusterConnection controller enterprise tests", func() { + var c client.Client + var ctx context.Context + var cfg *operatorv1.ManagementClusterConnection + var installation *operatorv1.Installation + var r reconcile.Reconciler + var clientScheme *runtime.Scheme + var mockStatus *status.MockStatus + var objTrackerWithCalls test.ObjectTrackerWithCalls + + notReady := &utils.ReadyFlag{} + ready := &utils.ReadyFlag{} + ready.MarkAsReady() + + BeforeEach(func() { + // Create a Kubernetes client. + clientScheme = runtime.NewScheme() + Expect(apis.AddToScheme(clientScheme, false)).ShouldNot(HaveOccurred()) + Expect(appsv1.SchemeBuilder.AddToScheme(clientScheme)).ShouldNot(HaveOccurred()) + Expect(rbacv1.SchemeBuilder.AddToScheme(clientScheme)).ShouldNot(HaveOccurred()) + err := operatorv1.SchemeBuilder.AddToScheme(clientScheme) + Expect(err).NotTo(HaveOccurred()) + objTrackerWithCalls = test.NewObjectTrackerWithCalls(clientScheme) + c = ctrlrfake.DefaultFakeClientBuilder(clientScheme).WithObjectTracker(&objTrackerWithCalls).Build() + ctx = context.Background() + mockStatus = &status.MockStatus{} + mockStatus.On("SetWarning", mock.Anything, mock.Anything).Return().Maybe() + mockStatus.On("ClearWarning", mock.Anything).Return().Maybe() + mockStatus.On("Run").Return() + mockStatus.On("AddDaemonsets", mock.Anything) + mockStatus.On("AddDeployments", mock.Anything) + mockStatus.On("AddStatefulSets", mock.Anything) + mockStatus.On("AddCronJobs", mock.Anything) + mockStatus.On("ClearDegraded", mock.Anything) + mockStatus.On("SetDegraded", mock.Anything, mock.Anything, mock.Anything, mock.Anything) + mockStatus.On("OnCRFound").Return() + mockStatus.On("ReadyToMonitor") + mockStatus.On("SetMetaData", mock.Anything).Return() + mockStatus.On("OnCRNotFound").Return() + + Expect(c.Create(ctx, &operatorv1.Monitor{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + })) + + Expect(c.Create(ctx, &v3.ClusterInformation{ObjectMeta: metav1.ObjectMeta{Name: "default"}})).NotTo(HaveOccurred()) + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + + Expect(c.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianNamespace}})) + certificateManager, err := certificatemanager.Create(c, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + Expect(c.Create(ctx, certificateManager.KeyPair().Secret(common.OperatorNamespace()))) // Persist the root-ca in the operator namespace. + secret, err := certificateManager.GetOrCreateKeyPair(c, render.GuardianSecretName, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + pcSecret, err := certificateManager.GetOrCreateKeyPair(c, render.PacketCaptureServerCert, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + promSecret, err := certificateManager.GetOrCreateKeyPair(c, monitor.PrometheusServerTLSSecretName, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + queryServerSecret, err := certificateManager.GetOrCreateKeyPair(c, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), []string{"a"}) + Expect(err).NotTo(HaveOccurred()) + + err = c.Create(ctx, secret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + err = c.Create(ctx, pcSecret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + err = c.Create(ctx, promSecret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + err = c.Create(ctx, queryServerSecret.Secret(common.OperatorNamespace())) + Expect(err).NotTo(HaveOccurred()) + + trustedBundle := certificateManager.CreateTrustedBundle() + Expect(c.Create(ctx, trustedBundle.ConfigMap(render.GuardianNamespace))).NotTo(HaveOccurred()) + + By("applying the required prerequisites") + // Create a ManagementClusterConnection in the k8s client. + cfg = &operatorv1.ManagementClusterConnection{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure", Generation: 3}, + Spec: operatorv1.ManagementClusterConnectionSpec{ + ManagementClusterAddr: "127.0.0.1:12345", + }, + } + err = c.Create(ctx, cfg) + Expect(err).NotTo(HaveOccurred()) + + installation = &operatorv1.Installation{ + Spec: operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "some.registry.org/", + }, + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Status: operatorv1.InstallationStatus{ + Variant: operatorv1.CalicoEnterprise, + Computed: &operatorv1.InstallationSpec{ + Registry: "my-reg", + KubernetesProvider: operatorv1.ProviderNone, + }, + }, + } + err = c.Create(ctx, installation) + Expect(err).NotTo(HaveOccurred()) + }) + + Context("image reconciliation", func() { + BeforeEach(func() { + Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + }) + + It("should use builtin images", func() { + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + d := appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianDeploymentName, + Namespace: render.GuardianNamespace, + }, + } + Expect(test.GetResource(c, &d)).To(BeNil()) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) + dexC := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) + Expect(dexC).ToNot(BeNil()) + Expect(dexC.Image).To(Equal( + fmt.Sprintf("some.registry.org/%s%s:%s", + components.TigeraImagePath, + components.ComponentTigeraCalico.Image, + components.ComponentTigeraCalico.Version))) + }) + It("should use images from imageset", func() { + Expect(c.Create(ctx, &operatorv1.ImageSet{ + ObjectMeta: metav1.ObjectMeta{Name: "enterprise-" + components.EnterpriseRelease}, + Spec: operatorv1.ImageSetSpec{ + Images: []operatorv1.Image{ + {Image: "tigera/calico", Digest: "sha256:guardianhash"}, + {Image: "tigera/calico", Digest: "sha256:deadbeef0123456789"}, + }, + }, + })).ToNot(HaveOccurred()) + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + d := appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianDeploymentName, + Namespace: render.GuardianNamespace, + }, + } + Expect(test.GetResource(c, &d)).To(BeNil()) + Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) + apiserver := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) + Expect(apiserver).ToNot(BeNil()) + Expect(apiserver.Image).To(Equal( + fmt.Sprintf("some.registry.org/%s%s@%s", + components.TigeraImagePath, + components.ComponentTigeraCalico.Image, + "sha256:guardianhash"))) + }) + }) + + Context("calico-system reconciliation", func() { + var licenseKey *v3.LicenseKey + BeforeEach(func() { + licenseKey = &v3.LicenseKey{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Status: v3.LicenseKeyStatus{ + Features: []string{ + common.TiersFeature, + common.EgressAccessControlFeature, + }, + }, + } + Expect(c.Create(ctx, licenseKey)).NotTo(HaveOccurred()) + Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) + }) + + Context("IP-based management cluster address", func() { + It("should render calico-system policy when tier and watch are ready", func() { + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + }) + + It("should omit calico-system policy and not degrade when tier is not ready", func() { + Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + + It("should degrade and wait when tier is ready, but tier watch is not ready", func() { + mockStatus = &status.MockStatus{} + mockStatus.On("Run").Return() + mockStatus.On("OnCRFound").Return() + mockStatus.On("SetMetaData", mock.Anything).Return() + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) + test.ExpectWaitForTierWatch(ctx, r, mockStatus) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + }) + + Context("Domain-based management cluster address", func() { + BeforeEach(func() { + cfg.Spec.ManagementClusterAddr = "mydomain.io:443" + Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) + }) + + It("should render calico-system policy when license and tier are ready", func() { + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + }) + + It("should render calico-system policy without domain-based egress when tier is ready, but license is not sufficient", func() { + licenseKey.Status.Features = []string{common.TiersFeature} + Expect(c.Update(ctx, licenseKey)).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + + // Verify no domain-based egress rules are present + for _, rule := range policies.Items[0].Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty(), + "Domain-based egress rules should not be present when license lacks EgressAccessControl") + } + }) + + It("should degrade and wait when tier and license are ready, but tier watch is not ready", func() { + mockStatus = &status.MockStatus{} + mockStatus.On("Run").Return() + mockStatus.On("OnCRFound").Return() + mockStatus.On("SetMetaData", mock.Anything).Return() + + r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) + test.ExpectWaitForTierWatch(ctx, r, mockStatus) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + + It("should render calico-system policy without domain-based egress when tier is ready but license is not ready", func() { + Expect(c.Delete(ctx, &v3.LicenseKey{ObjectMeta: metav1.ObjectMeta{Name: "default"}})).NotTo(HaveOccurred()) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + + // Verify no domain-based egress rules are present + for _, rule := range policies.Items[0].Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty(), + "Domain-based egress rules should not be present when license is not ready") + } + }) + + It("should omit calico-system policy when license is ready but tier is not ready", func() { + Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(0)) + }) + }) + + Context("Proxy detection", func() { + // Generate test cases based on the combinations of proxy address forms and proxy settings. + // Here we specify the base targets along with the base proxy IP, domain, and port that will be used for generation. + coreCases := generateCoreProxyTestCases("voltron.io:9000", "192.168.1.2:9000", "proxy.io", "10.1.2.3", "8080") + + // In case we support multiple guardian replicas in the future, we test specific multi-pod scenarios. + multiPodCases := multiplePodCases() + + testCases := append(coreCases, multiPodCases...) + for _, testCase := range testCases { + Describe(fmt.Sprintf("Proxy detection when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { + // Set up the test based on the test case. + BeforeEach(func() { + for i, proxy := range testCase.PodProxies { + createPodWithProxy(ctx, c, proxy, testCase.Lowercase, i) + } + + // Set the target + cfg.Spec.ManagementClusterAddr = testCase.Target + err := c.Update(ctx, cfg) + Expect(err).NotTo(HaveOccurred()) + }) + + It(fmt.Sprintf("detects proxy correctly when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { + // First reconcile creates the guardian deployment without any availability condition. + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + + // Validate that we made no calls to get Pods at this stage. + podGVR := schema.GroupVersionResource{ + Version: "v1", + Resource: "pods", + } + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(BeZero()) + + // Set the deployment to be unavailable. We need to recreate the deployment otherwise the status update is ignored. + gd := appsv1.Deployment{} + err = c.Get(ctx, client.ObjectKey{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, &gd) + Expect(err).NotTo(HaveOccurred()) + err = c.Delete(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + gd.ResourceVersion = "" + gd.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentAvailable, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.Time{Time: time.Now()}, + }} + err = c.Create(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + + // Reconcile again. We should see no calls since the deployment has not transitioned to available. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(0)) + + // Set the deployment to available. + err = c.Delete(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + gd.ResourceVersion = "" + gd.Status.Conditions = []appsv1.DeploymentCondition{{ + Type: appsv1.DeploymentAvailable, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.Time{Time: time.Now().Add(time.Minute)}, + }} + err = c.Create(ctx, &gd) + Expect(err).NotTo(HaveOccurred()) + + // Reconcile again. The proxy detection logic should kick in since the guardian deployment is ready. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) + + // Resolve the rendered rule that governs egress from guardian to voltron. + policies := v3.NetworkPolicyList{} + Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) + Expect(policies.Items).To(HaveLen(1)) + Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) + policy := policies.Items[0] + + // Generate the expectation based on the test case, and compare the rendered rule to our expectation. + expectedEgressRules := getExpectedEgressRulesFromCase(testCase) + Expect(policy.Spec.Egress).To(HaveLen(6 + len(expectedEgressRules))) + for i, egressRule := range expectedEgressRules { + managementClusterEgressRule := policy.Spec.Egress[5+i] + if egressRule.hostIsIP { + Expect(managementClusterEgressRule.Destination.Nets).To(HaveLen(1)) + Expect(managementClusterEgressRule.Destination.Nets[0]).To(Equal(fmt.Sprintf("%s/32", egressRule.host))) + Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) + } else { + Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{egressRule.host})) + Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) + } + } + + // Reconcile again. Verify that we do not cause any additional query for pods now that we have resolved the proxy. + _, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) + }) + }) + } + }) + }) + + val := []string{"some-value"} + DescribeTable("should render impersonation permissions correctly", func(impersonation *operatorv1.Impersonation, expectedUser, expectedGroup, expectedSA []string) { + By("ensuring a tigerastatus exists") + ts := &operatorv1.TigeraStatus{ + ObjectMeta: metav1.ObjectMeta{Name: "management-cluster-connection"}, + Spec: operatorv1.TigeraStatusSpec{}, + Status: operatorv1.TigeraStatusStatus{}, + } + Expect(c.Create(ctx, ts)).NotTo(HaveOccurred()) + + By("updating the CR with the impersonation settings, reconciling and fetching the results") + err := c.Get(ctx, client.ObjectKey{Name: cfg.Name, Namespace: cfg.Namespace}, cfg) + Expect(err).ShouldNot(HaveOccurred()) + cfg.Spec.Impersonation = impersonation + Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) + _, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ + Name: "management-cluster-connection", + Namespace: "", + }}) + Expect(err).ShouldNot(HaveOccurred()) + role := &rbacv1.ClusterRole{} + err = c.Get(ctx, client.ObjectKey{Name: render.GuardianClusterRoleName}, role) + Expect(err).NotTo(HaveOccurred()) + By("verifying the resulting RBAC") + var users, groups, sas []string + for _, rule := range role.Rules { + if len(rule.Verbs) == 1 && rule.Verbs[0] == "impersonate" { + if len(rule.Resources) == 1 { + switch rule.Resources[0] { + case "users": + users = rule.ResourceNames + case "groups": + groups = rule.ResourceNames + case "serviceaccounts": + sas = rule.ResourceNames + } + } + } + } + Expect(users).To(Equal(expectedUser)) + Expect(groups).To(Equal(expectedGroup)) + Expect(sas).To(Equal(expectedSA)) + }, + Entry("no impersonation configured", nil, nil, nil, nil), + Entry("all set", &operatorv1.Impersonation{Users: val, Groups: val, ServiceAccounts: val}, val, val, val), + Entry("all set to empty", &operatorv1.Impersonation{Users: []string{}, Groups: []string{}, ServiceAccounts: []string{}}, nil, nil, nil), + Entry("user set", &operatorv1.Impersonation{Users: val}, val, nil, nil), + Entry("groups set", &operatorv1.Impersonation{Groups: val}, nil, val, nil), + Entry("service accounts set", &operatorv1.Impersonation{ServiceAccounts: val}, nil, nil, val), + Entry("empty impersonation", &operatorv1.Impersonation{}, nil, nil, nil), + ) +}) diff --git a/pkg/controller/clusterconnection/clusterconnection_controller_test.go b/pkg/controller/clusterconnection/clusterconnection_controller_test.go index 401d62f9e4..66f37f8729 100644 --- a/pkg/controller/clusterconnection/clusterconnection_controller_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_controller_test.go @@ -21,7 +21,6 @@ import ( "net/url" "strconv" "strings" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -33,7 +32,6 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -43,7 +41,6 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/clusterconnection" "github.com/tigera/operator/pkg/controller/status" @@ -51,7 +48,6 @@ import ( ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/test" ) @@ -67,7 +63,6 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { var mockStatus *status.MockStatus var objTrackerWithCalls test.ObjectTrackerWithCalls - notReady := &utils.ReadyFlag{} ready := &utils.ReadyFlag{} ready.MarkAsReady() @@ -226,307 +221,6 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { }) }) - Context("image reconciliation", func() { - BeforeEach(func() { - Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - }) - - It("should use builtin images", func() { - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - d := appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: render.GuardianDeploymentName, - Namespace: render.GuardianNamespace, - }, - } - Expect(test.GetResource(c, &d)).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) - dexC := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) - Expect(dexC).ToNot(BeNil()) - Expect(dexC.Image).To(Equal( - fmt.Sprintf("some.registry.org/%s%s:%s", - components.TigeraImagePath, - components.ComponentTigeraCalico.Image, - components.ComponentTigeraCalico.Version))) - }) - It("should use images from imageset", func() { - Expect(c.Create(ctx, &operatorv1.ImageSet{ - ObjectMeta: metav1.ObjectMeta{Name: "enterprise-" + components.EnterpriseRelease}, - Spec: operatorv1.ImageSetSpec{ - Images: []operatorv1.Image{ - {Image: "tigera/calico", Digest: "sha256:guardianhash"}, - {Image: "tigera/calico", Digest: "sha256:deadbeef0123456789"}, - }, - }, - })).ToNot(HaveOccurred()) - - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - d := appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: render.GuardianDeploymentName, - Namespace: render.GuardianNamespace, - }, - } - Expect(test.GetResource(c, &d)).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) - apiserver := test.GetContainer(d.Spec.Template.Spec.Containers, render.GuardianContainerName) - Expect(apiserver).ToNot(BeNil()) - Expect(apiserver.Image).To(Equal( - fmt.Sprintf("some.registry.org/%s%s@%s", - components.TigeraImagePath, - components.ComponentTigeraCalico.Image, - "sha256:guardianhash"))) - }) - }) - - Context("calico-system reconciliation", func() { - var licenseKey *v3.LicenseKey - BeforeEach(func() { - licenseKey = &v3.LicenseKey{ - ObjectMeta: metav1.ObjectMeta{Name: "default"}, - Status: v3.LicenseKeyStatus{ - Features: []string{ - common.TiersFeature, - common.EgressAccessControlFeature, - }, - }, - } - Expect(c.Create(ctx, licenseKey)).NotTo(HaveOccurred()) - Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, ready, ready) - }) - - Context("IP-based management cluster address", func() { - It("should render calico-system policy when tier and watch are ready", func() { - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - }) - - It("should omit calico-system policy and not degrade when tier is not ready", func() { - Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - - It("should degrade and wait when tier is ready, but tier watch is not ready", func() { - mockStatus = &status.MockStatus{} - mockStatus.On("SetWarning", mock.Anything, mock.Anything).Return().Maybe() - mockStatus.On("ClearWarning", mock.Anything).Return().Maybe() - mockStatus.On("Run").Return() - mockStatus.On("OnCRFound").Return() - mockStatus.On("SetMetaData", mock.Anything).Return() - - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) - test.ExpectWaitForTierWatch(ctx, r, mockStatus) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - }) - - Context("Domain-based management cluster address", func() { - BeforeEach(func() { - cfg.Spec.ManagementClusterAddr = "mydomain.io:443" - Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) - }) - - It("should render calico-system policy when license and tier are ready", func() { - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - }) - - It("should render calico-system policy without domain-based egress when tier is ready, but license is not sufficient", func() { - licenseKey.Status.Features = []string{common.TiersFeature} - Expect(c.Update(ctx, licenseKey)).NotTo(HaveOccurred()) - - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - - // Verify no domain-based egress rules are present - for _, rule := range policies.Items[0].Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty(), - "Domain-based egress rules should not be present when license lacks EgressAccessControl") - } - }) - - It("should degrade and wait when tier and license are ready, but tier watch is not ready", func() { - mockStatus = &status.MockStatus{} - mockStatus.On("SetWarning", mock.Anything, mock.Anything).Return().Maybe() - mockStatus.On("ClearWarning", mock.Anything).Return().Maybe() - mockStatus.On("Run").Return() - mockStatus.On("OnCRFound").Return() - mockStatus.On("SetMetaData", mock.Anything).Return() - - r = clusterconnection.NewReconcilerWithShims(c, clientScheme, mockStatus, operatorv1.ProviderNone, notReady, ready) - test.ExpectWaitForTierWatch(ctx, r, mockStatus) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - - It("should render calico-system policy without domain-based egress when tier is ready but license is not ready", func() { - Expect(c.Delete(ctx, &v3.LicenseKey{ObjectMeta: metav1.ObjectMeta{Name: "default"}})).NotTo(HaveOccurred()) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - - // Verify no domain-based egress rules are present - for _, rule := range policies.Items[0].Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty(), - "Domain-based egress rules should not be present when license is not ready") - } - }) - - It("should omit calico-system policy when license is ready but tier is not ready", func() { - Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(0)) - }) - }) - - Context("Proxy detection", func() { - // Generate test cases based on the combinations of proxy address forms and proxy settings. - // Here we specify the base targets along with the base proxy IP, domain, and port that will be used for generation. - coreCases := generateCoreProxyTestCases("voltron.io:9000", "192.168.1.2:9000", "proxy.io", "10.1.2.3", "8080") - - // In case we support multiple guardian replicas in the future, we test specific multi-pod scenarios. - multiPodCases := multiplePodCases() - - testCases := append(coreCases, multiPodCases...) - for _, testCase := range testCases { - Describe(fmt.Sprintf("Proxy detection when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { - // Set up the test based on the test case. - BeforeEach(func() { - for i, proxy := range testCase.PodProxies { - createPodWithProxy(ctx, c, proxy, testCase.Lowercase, i) - } - - // Set the target - cfg.Spec.ManagementClusterAddr = testCase.Target - err := c.Update(ctx, cfg) - Expect(err).NotTo(HaveOccurred()) - }) - - It(fmt.Sprintf("detects proxy correctly when %+v", test.PrettyFormatProxyTestCase(testCase)), func() { - // First reconcile creates the guardian deployment without any availability condition. - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - // Validate that we made no calls to get Pods at this stage. - podGVR := schema.GroupVersionResource{ - Version: "v1", - Resource: "pods", - } - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(BeZero()) - - // Set the deployment to be unavailable. We need to recreate the deployment otherwise the status update is ignored. - gd := appsv1.Deployment{} - err = c.Get(ctx, client.ObjectKey{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, &gd) - Expect(err).NotTo(HaveOccurred()) - err = c.Delete(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - gd.ResourceVersion = "" - gd.Status.Conditions = []appsv1.DeploymentCondition{{ - Type: appsv1.DeploymentAvailable, - Status: v1.ConditionFalse, - LastTransitionTime: metav1.Time{Time: time.Now()}, - }} - err = c.Create(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - - // Reconcile again. We should see no calls since the deployment has not transitioned to available. - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(0)) - - // Set the deployment to available. - err = c.Delete(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - gd.ResourceVersion = "" - gd.Status.Conditions = []appsv1.DeploymentCondition{{ - Type: appsv1.DeploymentAvailable, - Status: v1.ConditionTrue, - LastTransitionTime: metav1.Time{Time: time.Now().Add(time.Minute)}, - }} - err = c.Create(ctx, &gd) - Expect(err).NotTo(HaveOccurred()) - - // Reconcile again. The proxy detection logic should kick in since the guardian deployment is ready. - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) - - // Resolve the rendered rule that governs egress from guardian to voltron. - policies := v3.NetworkPolicyList{} - Expect(c.List(ctx, &policies)).ToNot(HaveOccurred()) - Expect(policies.Items).To(HaveLen(1)) - Expect(policies.Items[0].Name).To(Equal("calico-system.guardian-access")) - policy := policies.Items[0] - - // Generate the expectation based on the test case, and compare the rendered rule to our expectation. - expectedEgressRules := getExpectedEgressRulesFromCase(testCase) - Expect(policy.Spec.Egress).To(HaveLen(6 + len(expectedEgressRules))) - for i, egressRule := range expectedEgressRules { - managementClusterEgressRule := policy.Spec.Egress[5+i] - if egressRule.hostIsIP { - Expect(managementClusterEgressRule.Destination.Nets).To(HaveLen(1)) - Expect(managementClusterEgressRule.Destination.Nets[0]).To(Equal(fmt.Sprintf("%s/32", egressRule.host))) - Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) - } else { - Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{egressRule.host})) - Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(egressRule.port))) - } - } - - // Reconcile again. Verify that we do not cause any additional query for pods now that we have resolved the proxy. - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(objTrackerWithCalls.CallCount(podGVR, test.ObjectTrackerCallList)).To(Equal(1)) - }) - }) - } - }) - }) - Context("Proxy setting", func() { DescribeTable("sets the proxy", func(http, https, noProxy bool) { installationCopy := installation.DeepCopy() @@ -780,57 +474,6 @@ var _ = Describe("ManagementClusterConnection controller tests", func() { }) }) - val := []string{"some-value"} - DescribeTable("should render impersonation permissions correctly", func(impersonation *operatorv1.Impersonation, expectedUser, expectedGroup, expectedSA []string) { - By("ensuring a tigerastatus exists") - ts := &operatorv1.TigeraStatus{ - ObjectMeta: metav1.ObjectMeta{Name: "management-cluster-connection"}, - Spec: operatorv1.TigeraStatusSpec{}, - Status: operatorv1.TigeraStatusStatus{}, - } - Expect(c.Create(ctx, ts)).NotTo(HaveOccurred()) - - By("updating the CR with the impersonation settings, reconciling and fetching the results") - err := c.Get(ctx, client.ObjectKey{Name: cfg.Name, Namespace: cfg.Namespace}, cfg) - Expect(err).ShouldNot(HaveOccurred()) - cfg.Spec.Impersonation = impersonation - Expect(c.Update(ctx, cfg)).NotTo(HaveOccurred()) - _, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{ - Name: "management-cluster-connection", - Namespace: "", - }}) - Expect(err).ShouldNot(HaveOccurred()) - role := &rbacv1.ClusterRole{} - err = c.Get(ctx, client.ObjectKey{Name: render.GuardianClusterRoleName}, role) - Expect(err).NotTo(HaveOccurred()) - By("verifying the resulting RBAC") - var users, groups, sas []string - for _, rule := range role.Rules { - if len(rule.Verbs) == 1 && rule.Verbs[0] == "impersonate" { - if len(rule.Resources) == 1 { - switch rule.Resources[0] { - case "users": - users = rule.ResourceNames - case "groups": - groups = rule.ResourceNames - case "serviceaccounts": - sas = rule.ResourceNames - } - } - } - } - Expect(users).To(Equal(expectedUser)) - Expect(groups).To(Equal(expectedGroup)) - Expect(sas).To(Equal(expectedSA)) - }, - Entry("no impersonation configured", nil, nil, nil, nil), - Entry("all set", &operatorv1.Impersonation{Users: val, Groups: val, ServiceAccounts: val}, val, val, val), - Entry("all set to empty", &operatorv1.Impersonation{Users: []string{}, Groups: []string{}, ServiceAccounts: []string{}}, nil, nil, nil), - Entry("user set", &operatorv1.Impersonation{Users: val}, val, nil, nil), - Entry("groups set", &operatorv1.Impersonation{Groups: val}, nil, val, nil), - Entry("service accounts set", &operatorv1.Impersonation{ServiceAccounts: val}, nil, nil, val), - Entry("empty impersonation", &operatorv1.Impersonation{}, nil, nil, nil), - ) }) func createPodWithProxy(ctx context.Context, c client.Client, config *test.ProxyConfig, lowercase bool, replicaNum int) { diff --git a/pkg/controller/clusterconnection/clusterconnection_suite_test.go b/pkg/controller/clusterconnection/clusterconnection_suite_test.go index 8967498282..2e6b6feb43 100644 --- a/pkg/controller/clusterconnection/clusterconnection_suite_test.go +++ b/pkg/controller/clusterconnection/clusterconnection_suite_test.go @@ -27,6 +27,7 @@ import ( func TestStatus(t *testing.T) { logf.SetLogger(zap.New(zap.WriteTo(ginkgo.GinkgoWriter))) gomega.RegisterFailHandler(ginkgo.Fail) + suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration() reporterConfig.JUnitReport = "../../../report/ut/clusterconnection_controller_suite.xml" ginkgo.RunSpecs(t, "pkg/controller/Management Cluster Connection Suite", suiteConfig, reporterConfig) diff --git a/pkg/controller/clusterconnection/shim_test.go b/pkg/controller/clusterconnection/shim_test.go index a792f930cf..494911dce5 100644 --- a/pkg/controller/clusterconnection/shim_test.go +++ b/pkg/controller/clusterconnection/shim_test.go @@ -25,6 +25,8 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -40,6 +42,7 @@ func NewReconcilerWithShims( ) reconcile.Reconciler { opts := options.ControllerOptions{ ShutdownContext: context.Background(), + Extensions: enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{}), Variant: operatorv1.CalicoEnterprise, } diff --git a/pkg/controller/gatewayapi/gatewayapi_controller.go b/pkg/controller/gatewayapi/gatewayapi_controller.go index 9ace267b28..07ad0001cc 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller.go @@ -182,7 +182,7 @@ type ReconcileGatewayAPI struct { clusterDomain string variant operatorv1.ProductVariant multiTenant bool - newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler + newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler watchEnvoyProxy func(namespacedName operatorv1.NamespacedName) error watchEnvoyGateway func(namespacedName operatorv1.NamespacedName) error watchGateways func() error diff --git a/pkg/controller/gatewayapi/gatewayapi_controller_test.go b/pkg/controller/gatewayapi/gatewayapi_controller_test.go index c3e2f403af..90f9fd6058 100644 --- a/pkg/controller/gatewayapi/gatewayapi_controller_test.go +++ b/pkg/controller/gatewayapi/gatewayapi_controller_test.go @@ -857,7 +857,7 @@ var _ = Describe("Gateway API controller tests", func() { var fakeComponentHandlers []*fakeComponentHandler -func FakeComponentHandler(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler { +func FakeComponentHandler(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler { h := &fakeComponentHandler{ client: client, scheme: scheme, diff --git a/pkg/controller/inputs.go b/pkg/controller/inputs.go new file mode 100644 index 0000000000..c9d4dba693 --- /dev/null +++ b/pkg/controller/inputs.go @@ -0,0 +1,46 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package controller holds the controller-phase inputs a reconcile hands to a +// variant extension. They live here rather than in the extensions package because +// they are what a controller gathers, not part of the extension mechanism. +package controller + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/render" +) + +// Name identifies the controller a ControllerExtension extends, so a variant can +// register a different hook per controller. +type Name string + +const ( + Installation Name = "installation" + Windows Name = "windows" + APIServer Name = "apiserver" + ClusterConnection Name = "clusterconnection" +) + +// Inputs is what a controller hands its variant extension: the render-phase inputs +// plus the deps needed to produce artifacts. The deps live here and not on +// render.Inputs so that modifiers, which only see render.Inputs, can't do I/O. +type Inputs struct { + RenderInputs render.Inputs + + Client client.Client + CertificateManager certificatemanager.CertificateManager +} diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 0d985d8b85..7f331d30b8 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -43,13 +43,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/event" + ctrl "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -58,13 +56,12 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" calicoclient "github.com/tigera/api/pkg/client/clientset_generated/clientset" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/active" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/common/discovery" - "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller" "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/controller/gatewayapi" "github.com/tigera/operator/pkg/controller/ippool" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration" @@ -75,30 +72,20 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/imports/admission" "github.com/tigera/operator/pkg/imports/crds" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/common/networkpolicy" - "github.com/tigera/operator/pkg/render/common/rbacmanagement" "github.com/tigera/operator/pkg/render/common/resourcequota" "github.com/tigera/operator/pkg/render/goldmane" "github.com/tigera/operator/pkg/render/kubecontrollers" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) const ( techPreviewFeatureSeccompApparmor = "tech-preview.operator.tigera.io/node-apparmor-profile" - - // The default port used by calico/node to report Calico Enterprise internal metrics. - // This is separate from the calico/node prometheus metrics port, which is user configurable. - defaultNodeReporterPort = 9081 - - defaultFelixMetricsDefaultPort = 9091 ) const InstallationName string = "calico" @@ -144,7 +131,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("failed to create Core Reconciler: %w", err) } - c, err := ctrlruntime.NewController("tigera-installation-controller", mgr, controller.Options{Reconciler: ri}) + c, err := ctrlruntime.NewController("tigera-installation-controller", mgr, ctrl.Options{Reconciler: ri}) if err != nil { return fmt.Errorf("failed to create tigera-installation-controller: %w", err) } @@ -216,13 +203,6 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } // Watch for changes to KubeControllersConfiguration. - // Watch GatewayAPI: spec.extensions.waf.state gates the WAF v3 surface on - // calico-kube-controllers. See design tigera/designs#25 (PMREQ-384) §Gating. - if err := c.WatchObject(&operatorv1.GatewayAPI{}, &handler.EnqueueRequestForObject{}); err != nil { - log.V(5).Info("Failed to create GatewayAPI watch", "err", err) - return fmt.Errorf("core-controller failed to watch operator GatewayAPI resource: %w", err) - } - err = c.WatchObject(&v3.KubeControllersConfiguration{}, &handler.EnqueueRequestForObject{}) if err != nil { return fmt.Errorf("tigera-installation-controller failed to watch KubeControllersConfiguration resource: %w", err) @@ -240,45 +220,13 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("tigera-installation-controller failed to watch BGPConfiguration resource: %w", err) } - if opts.Variant.IsEnterprise() { - // Watched so a toggle re-renders the access gated on it. - if err = utils.AddConfigMapWatch(c, rbacmanagement.ConfigMapName, common.CalicoNamespace, &handler.EnqueueRequestForObject{}); err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch ConfigMap %s: %w", rbacmanagement.ConfigMapName, err) - } - - // Watch for changes to primary resource ManagementCluster - err = c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // Watch for changes to primary resource ManagementClusterConnection - err = c.WatchObject(&operatorv1.ManagementClusterConnection{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // watch for change to primary resource LogCollector - err = c.WatchObject(&operatorv1.LogCollector{}, &handler.EnqueueRequestForObject{}) - if err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch primary resource: %v", err) - } - - // Watch the internal manager TLS secret in the operator namespace, which included in the bundle for es-kube-controllers. - if err = utils.AddSecretsWatch(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace()); err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch secret: %v", err) - } + if err = opts.Extensions.Installation().Watches(c); err != nil { + return fmt.Errorf("tigera-installation-controller failed to set up extension watches: %w", err) + } - if opts.ManageCRDs { - if err = addCRDWatches(c, operatorv1.CalicoEnterprise, opts.UseV3CRDs); err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch CRD resource: %v", err) - } - } - } else { - if opts.ManageCRDs { - if err = addCRDWatches(c, operatorv1.Calico, opts.UseV3CRDs); err != nil { - return fmt.Errorf("tigera-installation-controller failed to watch CRD resource: %v", err) - } + if opts.ManageCRDs { + if err = utils.AddCRDWatches(c, crds.GetCRDs(operatorv1.Calico, opts.UseV3CRDs)); err != nil { + return fmt.Errorf("tigera-installation-controller failed to watch CRD resource: %v", err) } } @@ -311,21 +259,6 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return nil } -func addCRDWatches(c ctrlruntime.Controller, v operatorv1.ProductVariant, useV3 bool) error { - pred := predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - // Create occurs because we've created it, so we can safely ignore it. - return false - }, - } - for _, x := range crds.GetCRDs(v, useV3) { - if err := c.WatchObject(x, &handler.EnqueueRequestForObject{}, pred); err != nil { - return err - } - } - return nil -} - // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager, opts options.ControllerOptions) (*ReconcileInstallation, error) { nm, err := migration.NewCoreNamespaceMigration(opts.K8sClientset) @@ -345,27 +278,18 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions) (*Reconc typhaScaler := newTyphaAutoscaler(opts.K8sClientset, nodeIndexInformer, typhaListWatch, statusManager) r := &ReconcileInstallation{ - config: mgr.GetConfig(), - client: mgr.GetClient(), - clientset: opts.K8sClientset, - scheme: mgr.GetScheme(), - shutdownContext: opts.ShutdownContext, - watches: make(map[runtime.Object]struct{}), - autoDetectedProvider: opts.DetectedProvider, - status: statusManager, - typhaAutoscaler: typhaScaler, - namespaceMigration: nm, - variant: opts.Variant, - clusterDomain: opts.ClusterDomain, - manageCRDs: opts.ManageCRDs, - multiTenant: opts.MultiTenant, - tierWatchReady: &utils.ReadyFlag{}, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, - v3CRDs: opts.UseV3CRDs, - kubernetesVersion: opts.KubernetesVersion, - apiDiscovery: opts.APIDiscovery, - cloud: opts.Cloud, + config: mgr.GetConfig(), + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + watches: make(map[runtime.Object]struct{}), + status: statusManager, + typhaAutoscaler: typhaScaler, + namespaceMigration: nm, + tierWatchReady: &utils.ReadyFlag{}, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, + opts: opts, + ext: opts.Extensions.Installation(), } r.status.Run(opts.ShutdownContext) r.typhaAutoscaler.start(opts.ShutdownContext) @@ -407,32 +331,20 @@ type ReconcileInstallation struct { // that reads objects from the cache and writes to the apiserver config *rest.Config client client.Client - clientset *kubernetes.Clientset scheme *runtime.Scheme - shutdownContext context.Context watches map[runtime.Object]struct{} - autoDetectedProvider operatorv1.Provider status status.StatusManager typhaAutoscaler *typhaAutoscaler typhaAutoscalerNonClusterHost *typhaAutoscaler namespaceMigration migration.NamespaceMigration - variant operatorv1.ProductVariant migrationChecked bool - clusterDomain string - manageCRDs bool - multiTenant bool tierWatchReady *utils.ReadyFlag migrationWatchReady *utils.ReadyFlag - v3CRDs bool - kubernetesVersion *common.VersionInfo - apiDiscovery *discovery.APIDiscovery - - // cloud indicates the operator is running as a Calico Cloud install. When false the calico - // kube-controllers render config leaves cloud behavior (e.g. the tesla image) off. - cloud bool + opts options.ControllerOptions + ext extensions.InstallationExtension // newComponentHandler returns a new component handler. Useful stub for unit testing. - newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler + newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...utils.ComponentHandlerOption) utils.ComponentHandler } // GetActivePools returns the full set of enabled IP pools in the cluster. @@ -879,7 +791,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // update Installation with defaults - if err := updateInstallationWithDefaults(ctx, r.client, instance, r.autoDetectedProvider, r.variant); err != nil { + if err := updateInstallationWithDefaults(ctx, r.client, instance, r.opts.DetectedProvider, r.opts.Variant); err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error querying installation", err, reqLogger) return reconcile.Result{}, err } @@ -943,7 +855,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // Update CRDs before persisting defaults. Defaulting can set a value only this operator version's // CRD accepts (e.g. an autodetected kubernetesProvider=Kind); on upgrade the old served CRD would // otherwise reject the write and the reconcile would loop before ever reaching the CRD update. - if err = r.updateCRDs(ctx, r.variant, reqLogger); err != nil { + if err = r.updateCRDs(ctx, r.opts.Variant, reqLogger); err != nil { return reconcile.Result{}, err } @@ -1048,37 +960,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - var managementCluster *operatorv1.ManagementCluster - var managementClusterConnection *operatorv1.ManagementClusterConnection - var logCollector *operatorv1.LogCollector - if r.variant.IsEnterprise() { - logCollector, err = utils.GetLogCollector(ctx, r.client) - if logCollector != nil { - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading LogCollector", err, reqLogger) - return reconcile.Result{}, err - } - } - - managementCluster, err = utils.GetManagementCluster(ctx, r.client) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementCluster", err, reqLogger) - return reconcile.Result{}, err - } - - managementClusterConnection, err = utils.GetManagementClusterConnection(ctx, r.client) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading ManagementClusterConnection", err, reqLogger) - return reconcile.Result{}, err - } - - if managementClusterConnection != nil && managementCluster != nil { - err = fmt.Errorf("having both a managementCluster and a managementClusterConnection is not supported") - r.status.SetDegraded(operatorv1.ResourceValidationError, "", err, reqLogger) - return reconcile.Result{}, err - } - } - includeV3NetworkPolicy := false // Ensure the calico-system tier exists, before rendering any network policies within it. // @@ -1097,7 +978,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } } - certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.clusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) + certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.opts.ClusterDomain, common.OperatorNamespace(), certificatemanager.WithLogger(reqLogger)) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -1110,18 +991,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - if instance.Spec.Variant.IsEnterprise() { - managerInternalTLSSecret, err := certificateManager.GetCertificate(r.client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, fmt.Sprintf("Error fetching TLS secret %s in namespace %s", render.ManagerInternalTLSSecretName, common.OperatorNamespace()), err, reqLogger) - return reconcile.Result{}, nil - } else if managerInternalTLSSecret != nil { - // It may seem odd to add the manager internal TLS secret to the trusted bundle for Typha / calico-node, but this bundle is also used - // for other components in this namespace such as es-kube-controllers, who communicates with Voltron and thus needs to trust this certificate. - typhaNodeTLS.TrustedBundle.AddCertificates(managerInternalTLSSecret) - } - } - birdTemplates, err := getBirdTemplates(r.client) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error retrieving confd templates", err, reqLogger) @@ -1222,63 +1091,29 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. - // Operator is responsible for creating a service which maps to that port. - // Here, we'll check the default felixconfiguration to see if the user is specifying - // a non-default port, and use that value if they are. - nodeReporterMetricsPort := defaultNodeReporterPort - var nodePrometheusTLS certificatemanagement.KeyPairInterface - calicoVersion := components.CalicoRelease + calicoVersion := r.ext.ProductVersion() - felixPrometheusMetricsPort := defaultFelixMetricsDefaultPort - - if instance.Spec.Variant.IsEnterprise() { - - // Determine the port to use for nodeReporter metrics. - if felixConfiguration.Spec.PrometheusReporterPort != nil { - nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort - } - if nodeReporterMetricsPort == 0 { - err := errors.New("felixConfiguration prometheusReporterPort=0 not supported") - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "invalid metrics port", err, reqLogger) - return reconcile.Result{}, err - } - - if felixConfiguration.Spec.PrometheusMetricsPort != nil { - felixPrometheusMetricsPort = *felixConfiguration.Spec.PrometheusMetricsPort - } - - nodePrometheusTLS, err = certificateManager.GetOrCreateKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - if nodePrometheusTLS != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) - } - prometheusClientCert, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, "Unable to fetch prometheus certificate", err, reqLogger) - return reconcile.Result{}, err - } - if prometheusClientCert != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(prometheusClientCert) - } - - // es-kube-controllers needs to trust the ESGW certificate. We'll fetch it here and add it to the trusted bundle. - // Note that although we're adding this to the typhaNodeTLS trusted bundle, it will be used by es-kube-controllers. This is because - // all components within this namespace share a trusted CA bundle. This is necessary because prior to v3.13 secrets were not signed by - // a single CA so we need to include each individually. - esgwCertificate, err := certificateManager.GetCertificate(r.client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve / validate %s", relasticsearch.PublicCertSecret), err, reqLogger) + ci := controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.opts.ClusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + }, + Client: r.client, + CertificateManager: certificateManager, + } + ci, extraKeyPairs, err := r.ext.ExtendInputs(ctx, ci) + if err != nil { + if reason, ok := extensions.DegradedReason(err); ok { + r.status.SetDegraded(reason, err.Error(), nil, reqLogger) + if reason == operatorv1.ResourceNotReady { + return reconcile.Result{}, nil + } return reconcile.Result{}, err } - if esgwCertificate != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(esgwCertificate) - } - - calicoVersion = components.EnterpriseRelease + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing installation extension", err, reqLogger) + return reconcile.Result{}, err } kubeControllersMetricsPort, err := utils.GetKubeControllerMetricsPort(ctx, r.client) @@ -1287,30 +1122,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - // Secure calico kube controller metrics. - var kubeControllerTLS certificatemanagement.KeyPairInterface - if instance.Spec.Variant.IsEnterprise() { - // Create or Get TLS certificates for kube controller. - kubeControllerTLS, err = certificateManager.GetOrCreateKeyPair( - r.client, - kubecontrollers.KubeControllerPrometheusTLSSecret, - common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error finding or creating TLS certificate kube controllers metric", err, reqLogger) - return reconcile.Result{}, err - } - - // Add prometheus client certificate to Trusted bundle. - kubeControllerPrometheusTLS, err := certificateManager.GetCertificate(r.client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get certificate for kube controllers", err, reqLogger) - return reconcile.Result{}, err - } else if kubeControllerPrometheusTLS != nil { - typhaNodeTLS.TrustedBundle.AddCertificates(kubeControllerTLS, kubeControllerPrometheusTLS) - } - } - nodeAppArmorProfile := "" a := instance.GetObjectMeta().GetAnnotations() if val, ok := a[techPreviewFeatureSeccompApparmor]; ok { @@ -1318,7 +1129,15 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } // Create a component handler to create or update the rendered components. - handler := r.newComponentHandler(log, r.client, r.scheme, instance) + handler := r.newComponentHandler( + log, + r.client, + r.scheme, + instance, + utils.WithModifier(func(c render.Component) render.Component { + return r.ext.Modify(c, ci.RenderInputs) + }), + ) // Render namespaces first - this ensures that any other controllers blocked on namespace existence can proceed. namespaceCfg := &render.NamespaceConfiguration{ @@ -1330,12 +1149,6 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile return reconcile.Result{}, err } - rbacManagementEnabled, err := utils.RBACManagementEnabled(ctx, r.client, instance.Spec.Variant, r.multiTenant) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceReadError, "Error reading the RBAC management UI ConfigMap", err, reqLogger) - return reconcile.Result{}, err - } - // Build the list of components to render, in rendering order. components := []render.Component{} if newActiveCM != nil && !installationMarkedForDeletion { @@ -1381,57 +1194,14 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } - // Read the GatewayAPI CR (if present) to decide whether to render the WAF - // v3 (Gateway API add-on) surface — env vars, RBAC, applicationlayer - // reconciler, and the in-process admission webhook — on - // calico-kube-controllers. Default-off: if no GatewayAPI CR exists or - // spec.extensions.waf.state != Enabled, the WAF surface is not rendered. - // See design tigera/designs#25 (PMREQ-384) §Gating. - wafGatewayExtensionEnabled := false - // gatewayAPIPresent means the GatewayAPI CR exists (regardless of waf.state), - // so the operator manages the Gateway API + Envoy Gateway CRDs the WAF - // reconcilers watch. It keeps the applicationlayer controller wired (with - // EnvoyExtensionPolicy delete RBAC) even while WAF is disabled, so the - // controller can tear down the EEPs it generated instead of being removed in - // the same reconcile that disables WAF (EV-6751). - gatewayAPIPresent := false - if gatewayAPI, msg, err := gatewayapi.GetGatewayAPI(ctx, r.client); err == nil { - gatewayAPIPresent = true - wafGatewayExtensionEnabled = gatewayAPI.Spec.IsWAFGatewayExtensionEnabled() - } else if !apierrors.IsNotFound(err) { - // Mirrors the GatewayAPI controller's handling: a read error or a - // duplicate default/tigera-secure pair degrades rather than guessing. - r.status.SetDegraded(operatorv1.ResourceReadError, msg, err, reqLogger) - return reconcile.Result{}, err - } - - // When the WAF v3 surface is enabled, issue the serving cert for the - // in-process WAF admission webhook (hosted by calico-kube-controllers, - // fronted by the tigera-waf-webhook Service). It is materialized into - // calico-system alongside the other kube-controllers certs below and mounted - // into the Pod by the kube-controllers render. - var wafWebhookTLS certificatemanagement.KeyPairInterface - if wafGatewayExtensionEnabled { - wafWebhookTLS, err = certificateManager.GetOrCreateKeyPair( - r.client, - applicationlayer.WAFWebhookServerTLSSecretName, - common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating WAF admission webhook TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - } - keyPairOptions := []rcertificatemanagement.KeyPairOption{ rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.NodeSecret, true, true), - rcertificatemanagement.NewKeyPairOption(nodePrometheusTLS, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecret, true, true), rcertificatemanagement.NewKeyPairOption(typhaNodeTLS.TyphaSecretNonClusterHost, true, true), - rcertificatemanagement.NewKeyPairOption(kubeControllerTLS, true, true), - // Nil when the WAF v3 surface is disabled; the certificate-management - // render skips nil key pairs. - rcertificatemanagement.NewKeyPairOption(wafWebhookTLS, true, true), + } + // Manage any key pairs the variant extension created controller-side. + for _, kp := range extraKeyPairs { + keyPairOptions = append(keyPairOptions, rcertificatemanagement.NewKeyPairOption(kp, true, true)) } components = append(components, @@ -1475,11 +1245,11 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile hepListWatch := cache.NewListWatchFromClient(calicoClient.ProjectcalicoV3().RESTClient(), "hostendpoints", corev1.NamespaceAll, fields.Everything()) hepIndexInformer := cache.NewSharedIndexInformer(hepListWatch, &v3.HostEndpoint{}, 0, cache.Indexers{}) - go hepIndexInformer.Run(r.shutdownContext.Done()) + go hepIndexInformer.Run(r.opts.ShutdownContext.Done()) - typhaNonClusterHostWatch := cache.NewListWatchFromClient(r.clientset.AppsV1().RESTClient(), "deployments", "calico-system", fields.OneTermEqualSelector("metadata.name", "calico-typha"+render.TyphaNonClusterHostSuffix)) - r.typhaAutoscalerNonClusterHost = newTyphaAutoscaler(r.clientset, hepIndexInformer, typhaNonClusterHostWatch, r.status, typhaAutoscalerOptionNonclusterHost(true)) - r.typhaAutoscalerNonClusterHost.start(r.shutdownContext) + typhaNonClusterHostWatch := cache.NewListWatchFromClient(r.opts.K8sClientset.AppsV1().RESTClient(), "deployments", "calico-system", fields.OneTermEqualSelector("metadata.name", "calico-typha"+render.TyphaNonClusterHostSuffix)) + r.typhaAutoscalerNonClusterHost = newTyphaAutoscaler(r.opts.K8sClientset, hepIndexInformer, typhaNonClusterHostWatch, r.status, typhaAutoscalerOptionNonclusterHost(true)) + r.typhaAutoscalerNonClusterHost.start(r.opts.ShutdownContext) } } } @@ -1491,7 +1261,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile Installation: &instance.Spec, TLS: typhaNodeTLS, MigrateNamespaces: needsNamespaceMigration, - ClusterDomain: r.clusterDomain, + ClusterDomain: r.opts.ClusterDomain, NonClusterHost: nonclusterhost, FelixHealthPort: *felixConfiguration.Spec.HealthPort, } @@ -1607,28 +1377,24 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // Build a configuration for rendering calico/node. nodeCfg := render.NodeConfiguration{ - GoldmaneRunning: goldmaneRunning, - K8sServiceEp: k8sapi.Endpoint, - Installation: &instance.Spec, - IPPools: crdPoolsToOperator(currentPools.Items), - LogCollector: logCollector, - BirdTemplates: birdTemplates, - TLS: typhaNodeTLS, - ClusterDomain: r.clusterDomain, - DefaultDNSPolicy: defaultDNSPolicy, - DefaultDNSConfig: defaultDNSConfig, - GoldmaneIP: goldmaneIP, - NodeReporterMetricsPort: nodeReporterMetricsPort, - BGPLayouts: bgpLayout, - NodeAppArmorProfile: nodeAppArmorProfile, - MigrateNamespaces: needsNamespaceMigration, - CanRemoveCNIFinalizer: canRemoveCNI, - PrometheusServerTLS: nodePrometheusTLS, - FelixHealthPort: *felixConfiguration.Spec.HealthPort, - NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, - FelixPrometheusMetricsEnabled: utils.IsFelixPrometheusMetricsEnabled(felixConfiguration), - FelixPrometheusMetricsPort: felixPrometheusMetricsPort, - V3CRDs: r.v3CRDs, + GoldmaneRunning: goldmaneRunning, + K8sServiceEp: k8sapi.Endpoint, + Installation: &instance.Spec, + IPPools: crdPoolsToOperator(currentPools.Items), + BirdTemplates: birdTemplates, + TLS: typhaNodeTLS, + ClusterDomain: r.opts.ClusterDomain, + DefaultDNSPolicy: defaultDNSPolicy, + DefaultDNSConfig: defaultDNSConfig, + GoldmaneIP: goldmaneIP, + BGPLayouts: bgpLayout, + NodeAppArmorProfile: nodeAppArmorProfile, + MigrateNamespaces: needsNamespaceMigration, + CanRemoveCNIFinalizer: canRemoveCNI, + FelixHealthPort: *felixConfiguration.Spec.HealthPort, + NodeCgroupV2Path: felixConfiguration.Spec.CgroupV2Path, + V3CRDs: r.opts.UseV3CRDs, + ImageOverrides: r.ext.Images(), } if bgpConfiguration.Spec.BindMode != nil { @@ -1672,58 +1438,17 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile } components = append(components, render.CSI(&csiCfg)) - // Build a configuration for rendering calico/kube-controllers. - // Provision a dedicated WAF wasm pull secret so the WAF reconciler - // replicates it into tenant namespaces without clashing with the - // operator-managed tigera-pull-secret the GatewayAPI render also copies - // there (EV-6386). The EnvoyExtensionPolicy image source takes a single - // pullSecretRef, so the registry auths of all Installation pull secrets - // are merged into it rather than picking one. - var wasmPullSecret *corev1.Secret - if wafGatewayExtensionEnabled && len(pullSecrets) > 0 { - var skipped []string - wasmPullSecret, skipped = kubecontrollers.MergeWAFPullSecret(pullSecrets) - if len(skipped) > 0 { - reqLogger.Info("Skipped unparseable imagePullSecrets when building the WAF wasm pull secret", "skipped", skipped) - } - } - // Provision the dedicated WAF wasm CA-bundle ConfigMap as a renamed copy of - // the trusted CA bundle, so the WAF reconciler replicates it into tenant - // namespaces for the Coraza wasm OCI registry TLS check without clashing with - // the operator-managed tigera-ca-bundle the GatewayAPI render also copies - // there (EV-6386). The dedicated source was previously a TODO; the full - // TrustedBundle (not the RO interface the kube-controllers render sees) is - // available here, so build it in the core controller. - var wasmCACert *corev1.ConfigMap - if wafGatewayExtensionEnabled { - wasmCACert = typhaNodeTLS.TrustedBundle.ConfigMap(common.CalicoNamespace) - wasmCACert.Name = kubecontrollers.WASMCACertName - } kubeControllersCfg := kubecontrollers.KubeControllersConfiguration{ - K8sServiceEp: k8sapi.Endpoint, - K8sServiceEpPodNetwork: k8sapi.PodNetworkEndpoint, - Installation: &instance.Spec, - ManagementCluster: managementCluster, - ManagementClusterConnection: managementClusterConnection, - ClusterDomain: r.clusterDomain, - MetricsPort: kubeControllersMetricsPort, - Terminating: installationMarkedForDeletion, - MetricsServerTLS: kubeControllerTLS, - TrustedBundle: typhaNodeTLS.TrustedBundle, - Namespace: common.CalicoNamespace, - BindingNamespaces: []string{common.CalicoNamespace}, - WAFGatewayExtensionEnabled: wafGatewayExtensionEnabled, - GatewayAPIPresent: gatewayAPIPresent, - WAFWebhookServerTLS: wafWebhookTLS, - WASMPullSecret: wasmPullSecret, - WASMCACert: wasmCACert, - // The webhook Service + ValidatingWebhookConfiguration are rendered by - // the kube-controllers component (and deleted when the WAF extension is - // disabled); the caBundle is the operator CA that issued the serving - // cert above. - WAFWebhookCABundle: certificateManager.KeyPair().GetCertificatePEM(), - Cloud: r.cloud, - RBACManagementEnabled: rbacManagementEnabled, + K8sServiceEp: k8sapi.Endpoint, + K8sServiceEpPodNetwork: k8sapi.PodNetworkEndpoint, + Installation: &instance.Spec, + ClusterDomain: r.opts.ClusterDomain, + MetricsPort: kubeControllersMetricsPort, + Terminating: installationMarkedForDeletion, + TrustedBundle: typhaNodeTLS.TrustedBundle, + Namespace: common.CalicoNamespace, + BindingNamespaces: []string{common.CalicoNamespace}, + ImageOverrides: r.ext.Images(), } components = append(components, kubecontrollers.NewCalicoKubeControllers(&kubeControllersCfg)) @@ -1859,13 +1584,15 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile r.status.ReadyToMonitor() // Check BYO certificate expiry warnings and propagate them to the status manager. - certificatemanagement.CheckKeyPairWarnings(map[string]certificatemanagement.KeyPairInterface{ + keyPairWarnings := map[string]certificatemanagement.KeyPairInterface{ render.TyphaTLSSecretName: typhaNodeTLS.TyphaSecret, render.NodeTLSSecretName: typhaNodeTLS.NodeSecret, render.TyphaTLSSecretName + render.TyphaNonClusterHostSuffix: typhaNodeTLS.TyphaSecretNonClusterHost, - render.NodePrometheusTLSServerSecret: nodePrometheusTLS, - kubecontrollers.KubeControllerPrometheusTLSSecret: kubeControllerTLS, - }, r.status) + } + for _, kp := range extraKeyPairs { + keyPairWarnings[kp.GetName()] = kp + } + certificatemanagement.CheckKeyPairWarnings(keyPairWarnings, r.status) // We can clear the degraded state now since as far as we know everything is in order. r.status.ClearDegraded() @@ -2113,36 +1840,13 @@ func (r *ReconcileInstallation) setDefaultsOnFelixConfiguration(ctx context.Cont updated = true } - if install.Spec.Variant.IsEnterprise() { - // Some platforms need a different default setting for dnsTrustedServers, because their DNS service is not named "kube-dns". - dnsService := "" - switch install.Spec.KubernetesProvider { - case operatorv1.ProviderOpenShift: - dnsService = "k8s-service:openshift-dns/dns-default" - case operatorv1.ProviderRKE2: - dnsService = "k8s-service:kube-system/rke2-coredns-rke2-coredns" - } - if dnsService != "" { - felixDefault := "k8s-service:kube-dns" - trustedServers := []string{dnsService} - // Keep any other values that are already configured, excepting the value - // that we are setting and the kube-dns default. - existingSetting := "" - if fc.Spec.DNSTrustedServers != nil { - existingSetting = strings.Join(*(fc.Spec.DNSTrustedServers), ",") - for _, server := range *(fc.Spec.DNSTrustedServers) { - if server != felixDefault && server != dnsService { - trustedServers = append(trustedServers, server) - } - } - } - newSetting := strings.Join(trustedServers, ",") - if newSetting != existingSetting { - fc.Spec.DNSTrustedServers = &trustedServers - updated = true - } - } + // Variant-specific FelixConfiguration defaults (e.g. the Enterprise + // provider-specific dnsTrustedServers) are owned by the variant extension. + extUpdated, err := r.ext.DefaultFelixConfiguration(&install.Spec, fc) + if err != nil { + return updated, err } + updated = updated || extUpdated // If BPF is enabled, but not set on FelixConfiguration, do so here. This could happen when an older // version of operator is replaced by the new one. Older versions of the operator used an @@ -2153,7 +1857,7 @@ func (r *ReconcileInstallation) setDefaultsOnFelixConfiguration(ctx context.Cont // If calico-node daemonset exists, we need to check the ENV VAR and set FelixConfiguration accordingly. // Otherwise, this is a fresh install in eBPF mode, set the felix config. ds := &appsv1.DaemonSet{} - err := r.client.Get(ctx, types.NamespacedName{Namespace: common.CalicoNamespace, Name: common.NodeDaemonSetName}, ds) + err = r.client.Get(ctx, types.NamespacedName{Namespace: common.CalicoNamespace, Name: common.NodeDaemonSetName}, ds) if err != nil { if !apierrors.IsNotFound(err) { reqLogger.Error(err, "An error occurred when getting the Daemonset resource") @@ -2366,10 +2070,10 @@ func (r *ReconcileInstallation) checkActive(log logr.Logger) (*corev1.ConfigMap, } func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operatorv1.ProductVariant, log logr.Logger) error { - if !r.manageCRDs { + if !r.opts.ManageCRDs { return nil } - crdComponent := render.NewCreationPassthrough(crds.ToRuntimeObjects(crds.GetCRDs(variant, r.v3CRDs)...)...) + crdComponent := render.NewCreationPassthrough(crds.ToRuntimeObjects(crds.GetCRDs(variant, r.opts.UseV3CRDs)...)...) // Specify nil for the CR so no ownership is put on the CRDs. We do this so removing the // Installation CR will not remove the CRDs. handler := r.newComponentHandler(log, r.client, r.scheme, nil) @@ -2381,19 +2085,19 @@ func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operator } func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Context, install *operatorv1.Installation, log logr.Logger) error { - if !r.manageCRDs || !r.v3CRDs { + if !r.opts.ManageCRDs || !r.opts.UseV3CRDs { return nil } // MutatingAdmissionPolicy served version was discovered once at startup (v1 was promoted to GA // in k8s 1.36 and v1beta1 (introduced in 1.32) is scheduled for removal in 1.37). - mapAPIVersion := r.apiDiscovery.ServedVersion(admission.APIGroup, admission.KindPolicy) + mapAPIVersion := r.opts.APIDiscovery.ServedVersion(admission.APIGroup, admission.KindPolicy) if mapAPIVersion == "" { r.status.SetDegraded(operatorv1.ResourceNotReady, "Kubernetes cluster does not serve MutatingAdmissionPolicy (requires v1.32+); policy defaulting will not be available", nil, log) return nil } - desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, mapAPIVersion) + desired := admission.GetMutatingAdmissionPolicies(install.Spec.Variant, r.opts.UseV3CRDs, mapAPIVersion) existingMAPs, existingMAPBs, err := admission.ListManaged(ctx, r.client, mapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed MutatingAdmissionPolicy resources", err, log) @@ -2404,20 +2108,20 @@ func (r *ReconcileInstallation) updateMutatingAdmissionPolicies(ctx context.Cont } func (r *ReconcileInstallation) updateValidatingAdmissionPolicies(ctx context.Context, install *operatorv1.Installation, log logr.Logger) error { - if !r.manageCRDs || !r.v3CRDs { + if !r.opts.ManageCRDs || !r.opts.UseV3CRDs { return nil } // ValidatingAdmissionPolicy reached GA (v1) well before MutatingAdmissionPolicy, so it has its own // served version and is reconciled independently of whether the cluster serves MAPs. If the cluster // doesn't serve it at all there's nothing to do, so skip rather than degrade. - vapAPIVersion := r.apiDiscovery.ServedVersion(admission.APIGroup, admission.KindValidatingPolicy) + vapAPIVersion := r.opts.APIDiscovery.ServedVersion(admission.APIGroup, admission.KindValidatingPolicy) if vapAPIVersion == "" { log.Info("Kubernetes cluster does not serve ValidatingAdmissionPolicy, skipping") return nil } - desired := admission.GetValidatingAdmissionPolicies(install.Spec.Variant, r.v3CRDs, vapAPIVersion) + desired := admission.GetValidatingAdmissionPolicies(install.Spec.Variant, r.opts.UseV3CRDs, vapAPIVersion) existingVAPs, existingVAPBs, err := admission.ListManagedValidating(ctx, r.client, vapAPIVersion) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Error listing managed ValidatingAdmissionPolicy resources", err, log) diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index 1c0acda980..b39cf7c801 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -36,7 +36,6 @@ import ( rbacv1 "k8s.io/api/rbac/v1" schedv1 "k8s.io/api/scheduling/v1" storagev1 "k8s.io/api/storage/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -57,6 +56,7 @@ import ( "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -192,18 +192,22 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - variant: operator.CalicoEnterprise, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + Variant: operator.CalicoEnterprise, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -395,6 +399,20 @@ var _ = Describe("Testing core-controller installation", func() { }) }) + It("degrades with a validation reason when the extension rejects the configuration", func() { + mockStatus.On("SetDegraded", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return() + + port := 0 + Expect(c.Create(ctx, &v3.FelixConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, + })).NotTo(HaveOccurred()) + + _, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).To(HaveOccurred()) + mockStatus.AssertCalled(GinkgoT(), "SetDegraded", operator.ResourceValidationError, "felixConfiguration prometheusReporterPort=0 not supported", mock.Anything, mock.Anything) + }) + Context("image tests", func() { It("should use builtin images", func() { _, err := r.Reconcile(ctx, reconcile.Request{}) @@ -821,19 +839,23 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - variant: operator.CalicoEnterprise, - migrationChecked: true, - clusterDomain: dns.DefaultClusterDomain, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + Variant: operator.CalicoEnterprise, + ClusterDomain: dns.DefaultClusterDomain, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -1043,18 +1065,22 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - variant: operator.CalicoEnterprise, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + Variant: operator.CalicoEnterprise, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -2246,7 +2272,7 @@ var _ = Describe("Testing core-controller installation", func() { cr.Spec.Variant = operator.Calico cr.Status.Variant = operator.Calico Expect(c.Create(ctx, cr)).NotTo(HaveOccurred()) - r.variant = operator.Calico + r.opts.Variant = operator.Calico Expect(c.Delete(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) _, err := r.Reconcile(ctx, reconcile.Request{}) @@ -2358,19 +2384,23 @@ var _ = Describe("Testing core-controller installation", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - variant: operator.CalicoEnterprise, - migrationChecked: true, - clusterDomain: dns.DefaultClusterDomain, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: utils.NewComponentHandler, + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + Variant: operator.CalicoEnterprise, + ClusterDomain: dns.DefaultClusterDomain, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: utils.NewComponentHandler, } r.typhaAutoscaler.start(ctx) @@ -2431,119 +2461,6 @@ var _ = Describe("Testing core-controller installation", func() { Expect(secret.GetOwnerReferences()).To(HaveLen(1)) }) - // The admin owns whether the ConfigMap exists and what it says; the operator only - // reads it. - Context("RBAC management UI feature gate", func() { - gateKey := client.ObjectKey{Name: rbacmanagement.ConfigMapName, Namespace: common.CalicoNamespace} - - // enabledControllers is where the gate's value is observable. - enabledControllers := func() string { - d := &appsv1.Deployment{} - Expect(c.Get(ctx, client.ObjectKey{ - Name: "calico-kube-controllers", Namespace: common.CalicoNamespace, - }, d)).ShouldNot(HaveOccurred()) - - container := test.GetContainer(d.Spec.Template.Spec.Containers, "calico-kube-controllers") - Expect(container).NotTo(BeNil()) - for _, env := range container.Env { - if env.Name == "ENABLED_CONTROLLERS" { - return env.Value - } - } - Fail("calico-kube-controllers has no ENABLED_CONTROLLERS env var") - return "" - } - - writeGate := func(value string) { - cm := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: gateKey.Name, Namespace: gateKey.Namespace}, - Data: map[string]string{rbacmanagement.ConfigMapKey: value}, - } - Expect(c.Create(ctx, cm)).ShouldNot(HaveOccurred()) - } - - It("does not create the ConfigMap", func() { - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - err = c.Get(ctx, gateKey, &corev1.ConfigMap{}) - Expect(apierrors.IsNotFound(err)).To(BeTrue(), "expected the operator not to create rbac-ui-config") - }) - - It("reads a missing ConfigMap as disabled", func() { - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - Expect(enabledControllers()).NotTo(ContainSubstring("rbacsync")) - }) - - It("follows the admin's value once they create the ConfigMap", func() { - writeGate("true") - - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - Expect(enabledControllers()).To(ContainSubstring("rbacsync")) - }) - - // Multi-tenant force-disables the feature on the ui-apis side. - It("withholds rbacsync on a multi-tenant management cluster even with the gate on", func() { - r.multiTenant = true - writeGate("true") - - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - Expect(enabledControllers()).NotTo(ContainSubstring("rbacsync")) - }) - - It("leaves the admin's value untouched across reconciles", func() { - writeGate("true") - - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - - cm := &corev1.ConfigMap{} - Expect(c.Get(ctx, gateKey, cm)).ShouldNot(HaveOccurred()) - Expect(cm.Data).To(HaveKeyWithValue(rbacmanagement.ConfigMapKey, "true")) - // Deleting the Installation must not take the admin's toggle with it. - Expect(cm.GetOwnerReferences()).To(BeEmpty()) - }) - - It("switches the feature back off when the admin deletes the ConfigMap", func() { - writeGate("true") - - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(enabledControllers()).To(ContainSubstring("rbacsync")) - - cm := &corev1.ConfigMap{} - Expect(c.Get(ctx, gateKey, cm)).ShouldNot(HaveOccurred()) - Expect(c.Delete(ctx, cm)).ShouldNot(HaveOccurred()) - - // Fail-closed, and the operator does not put the ConfigMap back. - _, err = r.Reconcile(ctx, reconcile.Request{}) - Expect(err).ShouldNot(HaveOccurred()) - Expect(enabledControllers()).NotTo(ContainSubstring("rbacsync")) - Expect(apierrors.IsNotFound(c.Get(ctx, gateKey, cm))).To(BeTrue()) - }) - - // An unreadable ConfigMap is unknown state, not absent, so it degrades rather - // than rendering as disabled. - It("degrades and requeues when the ConfigMap cannot be read", func() { - readErr := fmt.Errorf("the API server is having a bad day") - r.client = failingGateReadClient{Client: c, err: readErr} - mockStatus.On("SetDegraded", operator.ResourceReadError, - "Error reading the RBAC management UI ConfigMap", readErr.Error(), mock.Anything).Return().Once() - - _, err := r.Reconcile(ctx, reconcile.Request{}) - Expect(err).To(MatchError(readErr)) - // The shared mockStatus expects a full reconcile, which this returns early - // from, so assert the one call. - mockStatus.AssertCalled(GinkgoT(), "SetDegraded", operator.ResourceReadError, - "Error reading the RBAC management UI ConfigMap", readErr.Error(), mock.Anything) - }) - }) }) Context("with a fake component handler", func() { @@ -2609,18 +2526,22 @@ var _ = Describe("Testing core-controller installation", func() { componentHandler = newFakeComponentHandler() r = ReconcileInstallation{ - config: nil, // there is no fake for config - client: c, - scheme: scheme, - autoDetectedProvider: operator.ProviderNone, - status: mockStatus, - typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), - namespaceMigration: &fakeNamespaceMigration{}, - variant: operator.CalicoEnterprise, - migrationChecked: true, - tierWatchReady: ready, - migrationWatchReady: &utils.ReadyFlag{}, - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + Variant: operator.CalicoEnterprise, + }, + config: nil, // there is no fake for config + client: c, + scheme: scheme, + status: mockStatus, + typhaAutoscaler: newTyphaAutoscaler(cs, nodeIndexInformer, test.NewTyphaListWatch(cs), mockStatus), + namespaceMigration: &fakeNamespaceMigration{}, + migrationChecked: true, + tierWatchReady: ready, + migrationWatchReady: &utils.ReadyFlag{}, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2772,13 +2693,17 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1 MAPs when v1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2803,13 +2728,17 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1beta1 MAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Beta1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Beta1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2832,13 +2761,17 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should create v1alpha1 MAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Alpha1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Alpha1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2861,13 +2794,17 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when no served version exists and should set degraded", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(""), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(""), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2879,13 +2816,17 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when v3CRDs=false", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: false, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: false, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2896,13 +2837,17 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should not create MAPs when manageCRDs=false", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: false, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: false, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2926,13 +2871,17 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - client: clientFor(staleMAP, staleMAPB), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(staleMAP, staleMAPB), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2968,13 +2917,17 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - client: clientFor(initial...), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(initial...), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -2986,13 +2939,17 @@ var _ = Describe("updateMutatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3057,13 +3014,17 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1 VAPs when v1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3088,13 +3049,17 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1beta1 VAPs when only v1beta1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Beta1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Beta1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3117,13 +3082,17 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should create v1alpha1 VAPs when only v1alpha1 is served", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1Alpha1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1Alpha1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3134,13 +3103,17 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should skip without degrading when no served version exists", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(""), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(""), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3152,13 +3125,17 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should not create VAPs when v3CRDs=false", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: false, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: false, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3182,13 +3159,17 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { } r = ReconcileInstallation{ - client: clientFor(staleVAP, staleVAPB), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(staleVAP, staleVAPB), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3206,13 +3187,17 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { It("should work with Enterprise variant", func() { r = ReconcileInstallation{ - client: clientFor(), - scheme: scheme, - status: mockStatus, - manageCRDs: true, - v3CRDs: true, - apiDiscovery: discoveryFor(admission.VersionV1), - newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object) utils.ComponentHandler { + ext: testExtensions.Installation(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + ManageCRDs: true, + UseV3CRDs: true, + APIDiscovery: discoveryFor(admission.VersionV1), + }, + client: clientFor(), + scheme: scheme, + status: mockStatus, + newComponentHandler: func(logr.Logger, client.Client, *runtime.Scheme, metav1.Object, ...utils.ComponentHandlerOption) utils.ComponentHandler { return componentHandler }, } @@ -3223,17 +3208,3 @@ var _ = Describe("updateValidatingAdmissionPolicies", func() { Expect(componentHandler.objectsToCreate).To(HaveLen(2)) }) }) - -// failingGateReadClient fails the read of the gate ConfigMap and passes everything else -// through, to distinguish an unreadable ConfigMap from an absent one. -type failingGateReadClient struct { - client.Client - err error -} - -func (f failingGateReadClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { - if _, ok := obj.(*corev1.ConfigMap); ok && key.Name == rbacmanagement.ConfigMapName { - return f.err - } - return f.Client.Get(ctx, key, obj, opts...) -} diff --git a/pkg/controller/installation/installation_controller_suite_test.go b/pkg/controller/installation/installation_controller_suite_test.go index 4924f3468b..479710686c 100644 --- a/pkg/controller/installation/installation_controller_suite_test.go +++ b/pkg/controller/installation/installation_controller_suite_test.go @@ -25,8 +25,18 @@ import ( clientfeaturestesting "k8s.io/client-go/features/testing" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" ) +// testExtensions is the enterprise extension Set the installation controller +// tests reconcile with, mirroring how main wires it in production. Reconcilers +// built in these tests put it on their options so the node image overrides and +// modifiers apply. +var testExtensions = enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{}) + func TestInstallation(t *testing.T) { // Disable WatchListClient for tests. In client-go v0.35+, this feature defaults to true and // causes informers to wait for bookmark events that fake clients never send, leading to timeouts. diff --git a/pkg/controller/installation/windows_controller.go b/pkg/controller/installation/windows_controller.go index fafc671193..a38c820911 100644 --- a/pkg/controller/installation/windows_controller.go +++ b/pkg/controller/installation/windows_controller.go @@ -16,7 +16,6 @@ package installation import ( "context" - "errors" "fmt" "reflect" @@ -28,7 +27,7 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" + ctrl "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -43,6 +42,7 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/active" "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller" "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/options" @@ -50,10 +50,8 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" - "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/monitor" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var logw = logf.Log.WithName("controller_windows") @@ -66,7 +64,7 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e return fmt.Errorf("failed to create Windows Reconciler: %w", err) } - c, err := ctrlruntime.NewController("tigera-windows-controller", mgr, controller.Options{Reconciler: ri}) + c, err := ctrlruntime.NewController("tigera-windows-controller", mgr, ctrl.Options{Reconciler: ri}) if err != nil { return fmt.Errorf("failed to create tigera-windows-controller: %w", err) } @@ -82,7 +80,7 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e return fmt.Errorf("tigera-windows-controller failed to watch calico Tigerastatus: %w", err) } - if ri.autoDetectedProvider.IsOpenShift() { + if ri.opts.DetectedProvider.IsOpenShift() { // Watch for openshift network configuration as well. If we're running in OpenShift, we need to // merge this configuration with our own and the write back the status object. err = c.WatchObject(&configv1.Network{}, &handler.EnqueueRequestForObject{}) @@ -150,15 +148,8 @@ func AddWindowsController(mgr manager.Manager, opts options.ControllerOptions) e // Watch for changes to IPAMConfiguration. go utils.WaitToAddResourceWatch(c, opts.K8sClientset, logw, ri.ipamConfigWatchReady, []client.Object{&v3.IPAMConfiguration{TypeMeta: metav1.TypeMeta{Kind: v3.KindIPAMConfiguration}}}) - if ri.variant.IsEnterprise() { - for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { - if err = utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { - return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", render.NodePrometheusTLSServerSecret, ns, err) - } - if err = utils.AddSecretsWatch(c, monitor.PrometheusClientTLSSecretName, ns); err != nil { - return fmt.Errorf("tigera-windows-controller failed to watch secret '%s' in '%s' namespace: %w", monitor.PrometheusClientTLSSecretName, ns, err) - } - } + if err = opts.Extensions.Windows().Watches(c); err != nil { + return fmt.Errorf("tigera-windows-controller failed to set up extension watches: %w", err) } // Perform periodic reconciliation. This acts as a backstop to catch reconcile issues, @@ -176,11 +167,10 @@ type ReconcileWindows struct { client client.Client scheme *runtime.Scheme watches map[runtime.Object]struct{} - autoDetectedProvider operatorv1.Provider status status.StatusManager - variant operatorv1.ProductVariant - clusterDomain string ipamConfigWatchReady *utils.ReadyFlag + opts options.ControllerOptions + ext extensions.WindowsExtension } // newWindowsReconciler returns a new reconcile.Reconciler @@ -192,11 +182,10 @@ func newWindowsReconciler(mgr manager.Manager, opts options.ControllerOptions) ( client: mgr.GetClient(), scheme: mgr.GetScheme(), watches: make(map[runtime.Object]struct{}), - autoDetectedProvider: opts.DetectedProvider, status: statusManager, - variant: opts.Variant, - clusterDomain: opts.ClusterDomain, ipamConfigWatchReady: &utils.ReadyFlag{}, + opts: opts, + ext: opts.Extensions.Windows(), } r.status.Run(opts.ShutdownContext) return r, nil @@ -288,7 +277,7 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } - certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.clusterDomain, common.OperatorNamespace()) + certificateManager, err := certificatemanager.Create(r.client, &instance.Spec, r.opts.ClusterDomain, common.OperatorNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger) return reconcile.Result{}, err @@ -328,36 +317,9 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } } - // nodeReporterMetricsPort is a port used in Enterprise to host internal metrics. - // Operator is responsible for creating a service which maps to that port. - // Here, we'll check the default felixconfiguration to see if the user is specifying - // a non-default port, and use that value if they are. - nodeReporterMetricsPort := defaultNodeReporterPort - var nodePrometheusTLS certificatemanagement.KeyPairInterface - if instance.Spec.Variant.IsEnterprise() { - - // Determine the port to use for nodeReporter metrics. - if felixConfiguration.Spec.PrometheusReporterPort != nil { - nodeReporterMetricsPort = *felixConfiguration.Spec.PrometheusReporterPort - } - - if nodeReporterMetricsPort == 0 { - err := errors.New("felixConfiguration prometheusReporterPort=0 not supported") - r.status.SetDegraded(operatorv1.InvalidConfigurationError, "invalid metrics port", err, reqLogger) - return reconcile.Result{}, err - } - - // The key pair is created by the core controller, so if it isn't set, requeue to wait until it is - nodePrometheusTLS, err = certificateManager.GetKeyPair(r.client, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, r.clusterDomain)) - if err != nil { - r.status.SetDegraded(operatorv1.ResourceCreateError, "Error getting TLS certificate", err, reqLogger) - return reconcile.Result{}, err - } - } - var component render.Component - kubeDNSServiceName := utils.GetDNSServiceName(r.autoDetectedProvider) + kubeDNSServiceName := utils.GetDNSServiceName(r.opts.DetectedProvider) kubeDNSService := &corev1.Service{} err = r.client.Get(ctx, kubeDNSServiceName, kubeDNSService) if err != nil { @@ -377,15 +339,39 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ return reconcile.Result{}, err } + // Run the variant's windows controller extension to build the render inputs + // (creating no enterprise artifacts in core). + ci := controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: &instance.Spec, + FelixConfiguration: felixConfiguration, + ClusterDomain: r.opts.ClusterDomain, + TrustedBundle: typhaNodeTLS.TrustedBundle, + }, + Client: r.client, + CertificateManager: certificateManager, + } + ci, _, err = r.ext.ExtendInputs(ctx, ci) + if err != nil { + if reason, ok := extensions.DegradedReason(err); ok { + r.status.SetDegraded(reason, err.Error(), nil, reqLogger) + if reason == operatorv1.ResourceNotReady { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + r.status.SetDegraded(operatorv1.ResourceCreateError, "Error preparing windows extension", err, reqLogger) + return reconcile.Result{}, err + } + windowsCfg := render.WindowsConfiguration{ - K8sServiceEp: k8sapi.Endpoint, - K8sDNSServers: kubeDNSIPs, - Installation: &instance.Spec, - ClusterDomain: r.clusterDomain, - TLS: typhaNodeTLS, - PrometheusServerTLS: nodePrometheusTLS, - NodeReporterMetricsPort: nodeReporterMetricsPort, - VXLANVNI: *felixConfiguration.Spec.VXLANVNI, + K8sServiceEp: k8sapi.Endpoint, + K8sDNSServers: kubeDNSIPs, + Installation: &instance.Spec, + ClusterDomain: r.opts.ClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: *felixConfiguration.Spec.VXLANVNI, + ImageOverrides: r.ext.Images(), } component = render.Windows(&windowsCfg) @@ -406,7 +392,15 @@ func (r *ReconcileWindows) Reconcile(ctx context.Context, request reconcile.Requ } // Create a component handler to create or update the rendered components. - handler := utils.NewComponentHandler(logw, r.client, r.scheme, instance) + handler := utils.NewComponentHandler( + logw, + r.client, + r.scheme, + instance, + utils.WithModifier(func(c render.Component) render.Component { + return r.ext.Modify(c, ci.RenderInputs) + }), + ) if err := handler.CreateOrUpdateOrDelete(ctx, component, nil); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error creating / updating resource", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/controller/installation/windows_controller_test.go b/pkg/controller/installation/windows_controller_test.go index 2f6661d8b5..f0909c3bd0 100644 --- a/pkg/controller/installation/windows_controller_test.go +++ b/pkg/controller/installation/windows_controller_test.go @@ -24,11 +24,13 @@ import ( "github.com/stretchr/testify/mock" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operator "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" @@ -119,12 +121,16 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ + ext: testExtensions.Windows(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + Variant: operator.CalicoEnterprise, + }, config: nil, // there is no fake for config client: c, scheme: scheme, - autoDetectedProvider: operator.ProviderNone, status: mockStatus, - variant: operator.CalicoEnterprise, ipamConfigWatchReady: &utils.ReadyFlag{}, } r.ipamConfigWatchReady.MarkAsReady() @@ -155,7 +161,7 @@ var _ = Describe("windows-controller installation tests", func() { }, }, } - Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.autoDetectedProvider, r.variant)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.opts.DetectedProvider, r.opts.Variant)).NotTo(HaveOccurred()) certificateManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) Expect(err).NotTo(HaveOccurred()) prometheusTLS, err := certificateManager.GetOrCreateKeyPair(c, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace(), []string{monitor.PrometheusClientTLSSecretName}) @@ -194,7 +200,7 @@ var _ = Describe("windows-controller installation tests", func() { cr.Status = operator.InstallationStatus{ Variant: operator.Calico, } - Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.autoDetectedProvider, r.variant)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, cr, r.opts.DetectedProvider, r.opts.Variant)).NotTo(HaveOccurred()) // Set serviceCIDRs in the installation (required for Calico for Windows) cr.Spec.ServiceCIDRs = []string{"10.96.0.0/12"} @@ -609,12 +615,16 @@ var _ = Describe("windows-controller installation tests", func() { // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. r = ReconcileWindows{ + ext: testExtensions.Windows(), + opts: options.ControllerOptions{ + Extensions: testExtensions, + DetectedProvider: operator.ProviderNone, + Variant: operator.CalicoEnterprise, + }, config: nil, // there is no fake for config client: c, scheme: scheme, - autoDetectedProvider: operator.ProviderNone, status: mockStatus, - variant: operator.CalicoEnterprise, ipamConfigWatchReady: &utils.ReadyFlag{}, } r.ipamConfigWatchReady.MarkAsReady() @@ -663,7 +673,7 @@ var _ = Describe("windows-controller installation tests", func() { }, }, } - Expect(updateInstallationWithDefaults(ctx, r.client, instance, r.autoDetectedProvider, r.variant)).NotTo(HaveOccurred()) + Expect(updateInstallationWithDefaults(ctx, r.client, instance, r.opts.DetectedProvider, r.opts.Variant)).NotTo(HaveOccurred()) Expect(c.Create(ctx, instance)).NotTo(HaveOccurred()) }) AfterEach(func() { diff --git a/pkg/controller/logstorage/common/common.go b/pkg/controller/logstorage/common/common.go index 2c070439d7..92f34494a6 100644 --- a/pkg/controller/logstorage/common/common.go +++ b/pkg/controller/logstorage/common/common.go @@ -26,7 +26,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/crypto" - "github.com/tigera/operator/pkg/render/kubecontrollers" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" ) const ( @@ -45,7 +45,7 @@ const ( // the gateway credentials, and a secret containing real admin level credentials is created and stored in the tigera-elasticsearch namespace to be swapped in once // ES Gateway has confirmed that the gateway credentials match. func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1.Secret, esAdminUserName string, cli client.Client, h utils.NamespaceHelper) (*corev1.Secret, *corev1.Secret, *corev1.Secret, error) { - kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) + kubeControllersGatewaySecret, err := utils.GetSecret(ctx, cli, entkubecontrollers.ElasticsearchKubeControllersUserSecret, h.TruthNamespace()) if err != nil { return nil, nil, nil, err } @@ -53,11 +53,11 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 password := crypto.GeneratePassword(16) kubeControllersGatewaySecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, + Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: h.TruthNamespace(), }, Data: map[string][]byte{ - "username": []byte(kubecontrollers.ElasticsearchKubeControllersUserName), + "username": []byte(entkubecontrollers.ElasticsearchKubeControllersUserName), "password": []byte(password), }, } @@ -67,34 +67,34 @@ func CreateKubeControllersSecrets(ctx context.Context, esAdminUserSecret *corev1 return nil, nil, nil, err } - kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) + kubeControllersVerificationSecret, err := utils.GetSecret(ctx, cli, entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersVerificationSecret == nil { kubeControllersVerificationSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, + Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, }, }, Data: map[string][]byte{ - "username": []byte(kubecontrollers.ElasticsearchKubeControllersUserName), + "username": []byte(entkubecontrollers.ElasticsearchKubeControllersUserName), "password": hashedPassword, }, } } - kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) + kubeControllersSecureUserSecret, err := utils.GetSecret(ctx, cli, entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, h.InstallNamespace()) if err != nil { return nil, nil, nil, err } if kubeControllersSecureUserSecret == nil { kubeControllersSecureUserSecret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, + Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: h.InstallNamespace(), Labels: map[string]string{ ESGatewaySelectorLabel: ESGatewaySelectorLabelValue, diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go index 735685963c..669bd85ce8 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go @@ -44,6 +44,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils" "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/render" "github.com/tigera/operator/pkg/render/common/cloudconfig" "github.com/tigera/operator/pkg/render/common/networkpolicy" @@ -145,7 +146,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if err := utils.AddDeploymentWatch(c, esgateway.DeploymentName, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } - if err := utils.AddDeploymentWatch(c, kubecontrollers.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { + if err := utils.AddDeploymentWatch(c, entkubecontrollers.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } if opts.Cloud && opts.ElasticExternal { @@ -179,7 +180,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { // Start goroutines to establish watches against projectcalico.org/v3 resources. go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, r.tierWatchReady) go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, []types.NamespacedName{ - {Name: kubecontrollers.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, + {Name: entkubecontrollers.EsKubeControllerNetworkPolicyName, Namespace: esKubeControllersNamespace.InstallNamespace()}, }) return nil @@ -273,7 +274,7 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec // Get secrets needed for kube-controllers to talk to elastic. This is needed for zero-tenants and single-tenants // that deploy es-kube-controllers and need to talk to es-gateway var kubeControllersUserSecret *core.Secret - kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, kubecontrollers.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) + kubeControllersUserSecret, err = utils.GetSecret(ctx, r.client, entkubecontrollers.ElasticsearchKubeControllersUserSecret, helper.TruthNamespace()) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get kube controllers gateway secret", err, reqLogger) return reconcile.Result{}, err @@ -349,7 +350,6 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec ClusterDomain: r.clusterDomain, Authentication: authentication, KubeControllersGatewaySecret: kubeControllersUserSecret, - LogStorageExists: logStorage != nil, TrustedBundle: trustedBundle, Namespace: helper.InstallNamespace(), BindingNamespaces: namespaces, @@ -361,7 +361,7 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec return result, err } } - esKubeControllerComponents := kubecontrollers.NewElasticsearchKubeControllers(&kubeControllersCfg) + esKubeControllerComponents := entkubecontrollers.NewElasticsearchKubeControllers(&kubeControllersCfg) imageSet, err := imageset.GetImageSet(ctx, r.client, r.variant) if err != nil { diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go index 1052bd148f..51d9569239 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers_test.go @@ -46,8 +46,8 @@ import ( "github.com/tigera/operator/pkg/controller/utils" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/logstorage" "github.com/tigera/operator/pkg/render/logstorage/esgateway" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -239,7 +239,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: entkubecontrollers.EsKubeController, Namespace: common.CalicoNamespace, }, } @@ -279,12 +279,12 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: entkubecontrollers.EsKubeController, Namespace: common.CalicoNamespace, }, } Expect(test.GetResource(cli, &dep)).To(BeNil()) - kc := test.GetContainer(dep.Spec.Template.Spec.Containers, kubecontrollers.EsKubeController) + kc := test.GetContainer(dep.Spec.Template.Spec.Containers, entkubecontrollers.EsKubeController) Expect(kc).ToNot(BeNil()) Expect(kc.Image).To(Equal(fmt.Sprintf("some.registry.org/%s%s@%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, "sha256:kubecontrollershash"))) }) @@ -329,7 +329,7 @@ var _ = Describe("LogStorage ES kube-controllers controller", func() { dep := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: kubecontrollers.EsKubeController, + Name: entkubecontrollers.EsKubeController, Namespace: common.CalicoNamespace, }, } diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index 7a7f0d8622..dc9fb15f8a 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -20,6 +20,7 @@ import ( v1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/common/discovery" + "github.com/tigera/operator/pkg/extensions" "k8s.io/client-go/kubernetes" ) @@ -68,4 +69,8 @@ type ControllerOptions struct { // the operator cares about. Populated once at startup so controllers can branch on API // availability without issuing further discovery requests at reconcile time. APIDiscovery *discovery.APIDiscovery + + // Extensions are the variant extensions the operator runs with, for the Variant + // above. The core operator leaves them unset and runs the base behavior. + Extensions extensions.Extensions } diff --git a/pkg/controller/utils/component.go b/pkg/controller/utils/component.go index f705df4152..0284ff4d20 100644 --- a/pkg/controller/utils/component.go +++ b/pkg/controller/utils/component.go @@ -27,6 +27,7 @@ import ( netv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" + apiextenv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1" kbv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/kibana/v1" @@ -42,12 +43,17 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apigroup" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" ) @@ -79,16 +85,32 @@ type ComponentHandler interface { SetCreateOnly() } +// ComponentHandlerOption configures a componentHandler. +type ComponentHandlerOption func(*componentHandler) + +// ComponentModifier post-processes a component before the handler renders it. The +// handler applies it to every component, and never learns what it does. +type ComponentModifier func(render.Component) render.Component + +// WithModifier supplies the modifier the handler runs each component through. +func WithModifier(m ComponentModifier) ComponentHandlerOption { + return func(c *componentHandler) { c.modify = m } +} + // cr is allowed to be nil in the case we don't want to put ownership on a resource, // this is useful for CRD management so that they are not removed automatically. -func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object) ComponentHandler { - return &componentHandler{ +func NewComponentHandler(log logr.Logger, cli client.Client, scheme *runtime.Scheme, cr metav1.Object, opts ...ComponentHandlerOption) ComponentHandler { + h := &componentHandler{ client: cli, scheme: scheme, cr: cr, log: log, apiGroupEnvs: apigroup.EnvVars(), } + for _, o := range opts { + o(h) + } + return h } type componentHandler struct { @@ -98,6 +120,7 @@ type componentHandler struct { log logr.Logger createOnly bool apiGroupEnvs []v1.EnvVar + modify ComponentModifier } func (c *componentHandler) SetCreateOnly() { @@ -440,6 +463,10 @@ func resetMetadataForCreate(obj client.Object) { } func (c *componentHandler) CreateOrUpdateOrDelete(ctx context.Context, component render.Component, status status.StatusManager) error { + if c.modify != nil { + component = c.modify(component) + } + // Before creating the component, make sure that it is ready. This provides a hook to do // dependency checking for the component. cmpLog := c.log.WithValues("component", reflect.TypeOf(component)) @@ -1173,7 +1200,6 @@ func addComponentLabel(obj metav1.Object, cr metav1.Object) { owner, ok := cr.(runtime.Object) if ok && owner.GetObjectKind() != nil && owner.GetObjectKind() != nil { obj.GetLabels()["app.kubernetes.io/component"] = sanitizeLabel(owner.GetObjectKind().GroupVersionKind().GroupKind().String()) - } } } @@ -1279,3 +1305,20 @@ func policyManagementDisabled(installation *operatorv1.InstallationSpec) bool { installation.NetworkPolicy.ManagePolicies != nil && *installation.NetworkPolicy.ManagePolicies == operatorv1.NetworkPolicyManagementDisabled } + +// AddCRDWatches watches the given CRDs, so the operator notices a managed CRD being +// changed out from under it. A variant extension calls it for the CRDs it adds. +func AddCRDWatches(c ctrlruntime.Controller, defs []*apiextenv1.CustomResourceDefinition) error { + pred := predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + // Create occurs because we've created it, so we can safely ignore it. + return false + }, + } + for _, x := range defs { + if err := c.WatchObject(x, &handler.EnqueueRequestForObject{}, pred); err != nil { + return err + } + } + return nil +} diff --git a/pkg/controller/utils/component_enterprise_test.go b/pkg/controller/utils/component_enterprise_test.go new file mode 100644 index 0000000000..b013075403 --- /dev/null +++ b/pkg/controller/utils/component_enterprise_test.go @@ -0,0 +1,85 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" + "github.com/tigera/operator/pkg/render" +) + +// A real typha component goes through the handler with the enterprise modifier +// attached, so this fails if dispatch or the modifier stops matching render. +var _ = Describe("enterprise typha modifier integration", func() { + It("applies the enterprise typha modifier to real render output", func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certManager, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + nodeKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + typhaKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.TyphaCommonName}) + Expect(err).NotTo(HaveOccurred()) + + instance := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{Type: operatorv1.PluginCalico}, + } + typhaCfg := &render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + TLS: &render.TyphaNodeTLS{ + TrustedBundle: certManager.CreateTrustedBundle(), + TyphaSecret: typhaKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + }, + } + + ext := enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{}).Installation() + renderInputs := render.Inputs{Installation: instance} + handler := utils.NewComponentHandler(logf.Log, cli, scheme, nil, utils.WithModifier(func(c render.Component) render.Component { + return ext.Modify(c, renderInputs) + })) + Expect(handler.CreateOrUpdateOrDelete(context.Background(), render.Typha(typhaCfg), nil)).NotTo(HaveOccurred()) + + role := &rbacv1.ClusterRole{} + Expect(cli.Get(context.Background(), client.ObjectKey{Name: "calico-typha"}, role)).NotTo(HaveOccurred()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + }) +}) diff --git a/pkg/enterprise/apiserver/extension.go b/pkg/enterprise/apiserver/extension.go new file mode 100644 index 0000000000..c1c6c5ce4e --- /dev/null +++ b/pkg/enterprise/apiserver/extension.go @@ -0,0 +1,1837 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiserver + +import ( + "context" + "fmt" + "net/url" + "reflect" + "slices" + "strings" + + admregv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/controller/utils/imageset" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/dns" + eoptions "github.com/tigera/operator/pkg/enterprise/options" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/authentication" + rcomp "github.com/tigera/operator/pkg/render/common/components" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/rbacmanagement" + "github.com/tigera/operator/pkg/render/common/securitycontext" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +const ( + auditLogsVolumeName = "calico-audit-logs" + auditPolicyVolumeName = "calico-audit-policy" + + // linseedAccessClusterRoleName is the Enterprise, multi-tenant-only ClusterRole granting + // each tenant's calico-apiserver identity read access to Linseed policy activity data. + linseedAccessClusterRoleName = "calico-apiserver-linseed-access" +) + +// apiServerRenderData is the controller-produced data the API server hook hands to its +// modifiers through Inputs.Extension. It carries the enterprise inputs the base +// render no longer knows about: the management cluster / managed cluster CRs, the +// ApplicationLayer (which drives the L7 sidecar), the cert-management-only query server +// keypair, the OIDC key validator config, and the resolved L7 sidecar images. +type apiServerRenderData struct { + managementCluster *operatorv1.ManagementCluster + managementClusterConnection *operatorv1.ManagementClusterConnection + applicationLayer *operatorv1.ApplicationLayer + queryServerTLS certificatemanagement.KeyPairInterface + keyValidatorConfig authentication.KeyValidatorConfig + l7EnvoyImage string + dikastesImage string + cloud bool + + // calicoImage is the image the query server and L7 admission controller containers + // run. The base render resolves it too, but a modifier runs with no ImageSet. + calicoImage string + + // rbacManagementEnabled mirrors the rbac-ui-config gate. ui-apis writes role bindings + // impersonating the caller, so tigera-network-admin needs the verbs itself. + rbacManagementEnabled bool + + // bindingNamespaces is the set of tenant namespaces whose calico-apiserver ServiceAccount + // should be granted Linseed access. Empty for zero/single-tenant clusters; every tenant + // namespace for multi-tenant management clusters. + bindingNamespaces []string +} + +// apiServerData pulls the API server hook's render data back out of the render inputs, +// returning the zero value when none is set. +func apiServerData(ri render.Inputs) apiServerRenderData { + return render.ExtractExtensionData[apiServerRenderData](ri) +} + +// apiServer carries the rendered API server configuration, the resolved image, and the +// controller-produced render data so the enterprise builders can construct the +// Enterprise-only objects and deployment additions. +type apiServer struct { + cfg *render.APIServerConfiguration + calicoImage string + data apiServerRenderData +} + +// Extension is the Calico Enterprise behavior for the API server controller. +type Extension struct { + variant operatorv1.ProductVariant + opts eoptions.Options +} + +var _ extensions.APIServerExtension = &Extension{} + +// New returns the API server extension for the variant the operator resolved. +func New(variant operatorv1.ProductVariant, opts eoptions.Options) *Extension { + return &Extension{variant: variant, opts: opts} +} + +// Modify dispatches over the components the API server controller renders. +func (e *Extension) Modify(c render.Component, ri render.Inputs) render.Component { + switch t := c.(type) { + case render.APIServerComponent: + return extensions.Decorate(c, ri, e.variant, func(create, del []client.Object) ([]client.Object, []client.Object) { + return modifyAPIServer(ri, t.APIServerConfig(), create, del) + }) + case render.APIServerPolicyComponent: + return extensions.Decorate(c, ri, e.variant, func(create, del []client.Object) ([]client.Object, []client.Object) { + return modifyAPIServerPolicy(ri, t.APIServerPolicyConfig(), create, del) + }) + default: + return c + } +} + +func (c *apiServer) isSidecarInjectionEnabled() bool { + al := c.data.applicationLayer + return al != nil && + al.Spec.SidecarInjection != nil && + *al.Spec.SidecarInjection == operatorv1.SidecarEnabled +} + +// Watches registers the enterprise resources the API server controller reconciles on. +func (e *Extension) Watches(c ctrlruntime.Controller) error { + for _, obj := range []client.Object{ + &operatorv1.ApplicationLayer{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}}, + &operatorv1.ManagementCluster{}, + &operatorv1.ManagementClusterConnection{}, + &operatorv1.Authentication{}, + } { + if err := c.WatchObject(obj, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + } + // The switch gating the RBAC management UI rules. + if err := utils.AddConfigMapWatch(c, rbacmanagement.ConfigMapName, common.CalicoNamespace, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + for _, namespace := range []string{common.OperatorNamespace(), render.APIServerNamespace} { + for _, secretName := range []string{render.VoltronTunnelSecretName, render.ManagerTLSSecretName} { + if err := utils.AddSecretsWatch(c, secretName, namespace); err != nil { + return err + } + } + } + return utils.AddSecretsWatch(c, render.VoltronLinseedPublicCert, common.OperatorNamespace()) +} + +// ExtendInputs does the enterprise controller-side work: it builds the trusted bundle, +// fetches the enterprise CRs, creates the query server certificate, resolves the L7 +// sidecar images, and stashes them for the modifiers. The base API server render carries +// none of this. +func (e *Extension) ExtendInputs(ctx context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) { + in := ci.RenderInputs.Installation + + trustedBundle, err := ci.CertificateManager.CreateNamedTrustedBundleFromSecrets(render.APIServerResourceName, ci.Client, common.OperatorNamespace(), false) + if err != nil { + return ci, nil, fmt.Errorf("unable to create the trusted bundle: %w", err) + } + + applicationLayer, err := utils.GetApplicationLayer(ctx, ci.Client) + if err != nil { + return ci, nil, fmt.Errorf("error reading ApplicationLayer: %w", err) + } + + managementCluster, err := utils.GetManagementCluster(ctx, ci.Client) + if err != nil { + return ci, nil, fmt.Errorf("error reading ManagementCluster: %w", err) + } + + managementClusterConnection, err := utils.GetManagementClusterConnection(ctx, ci.Client) + if err != nil { + return ci, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) + } + + if managementCluster != nil && managementClusterConnection != nil { + return ci, nil, extensions.InvalidConfigf("having both a ManagementCluster and a ManagementClusterConnection is not supported") + } + + rbacManagementEnabled, err := utils.RBACManagementEnabled(ctx, ci.Client, e.variant, e.opts.MultiTenant) + if err != nil { + return ci, nil, fmt.Errorf("error reading the RBAC management UI ConfigMap: %w", err) + } + + // Management cluster only: the apiserver mounts the tunnel CA secret so it can sign + // certificates for managed clusters. The manager controller writes it once + // ManagementCluster.Spec.TLS is defaulted; degrade until it exists. + if managementCluster != nil && managementCluster.Spec.TLS != nil && !e.opts.MultiTenant { + if _, err := utils.GetSecret(ctx, ci.Client, managementCluster.Spec.TLS.SecretName, common.OperatorNamespace()); err != nil { + return ci, nil, fmt.Errorf("unable to fetch the tunnel secret: %w", err) + } + } + + prometheusCertificate, err := ci.CertificateManager.GetCertificate(ci.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + if err != nil { + return ci, nil, fmt.Errorf("failed to get certificate: %w", err) + } + if prometheusCertificate != nil { + trustedBundle.AddCertificates(prometheusCertificate) + } + + if managementClusterConnection != nil { + voltronLinseedCert, err := ci.CertificateManager.GetCertificate(ci.Client, render.VoltronLinseedPublicCert, common.OperatorNamespace()) + if err != nil { + return ci, nil, fmt.Errorf("failed to retrieve %s: %w", render.VoltronLinseedPublicCert, err) + } + if voltronLinseedCert != nil { + trustedBundle.AddCertificates(voltronLinseedCert) + } + } + + // Authentication: when a Dex-backed Authentication CR is ready, add its cert to the + // bundle and build the key validator config for the query server and the policy. + var keyValidatorConfig authentication.KeyValidatorConfig + authenticationCR, err := utils.GetAuthentication(ctx, ci.Client) + if err != nil && !apierrors.IsNotFound(err) { + return ci, nil, extensions.Degradedf(operatorv1.ResourceReadError, "error while fetching Authentication: %s", err) + } + if authenticationCR != nil && authenticationCR.Status.State == operatorv1.TigeraStatusReady { + if utils.DexEnabled(authenticationCR) { + certificate, err := ci.CertificateManager.GetCertificate(ci.Client, render.DexTLSSecretName, common.OperatorNamespace()) + if err != nil { + return ci, nil, extensions.Degradedf(operatorv1.CertificateError, "failed to retrieve %s: %s", render.DexTLSSecretName, err) + } else if certificate == nil { + return ci, nil, extensions.NotReadyf("waiting for secret '%s' to become available", render.DexTLSSecretName) + } + trustedBundle.AddCertificates(certificate) + } + keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, ci.Client, authenticationCR, ci.RenderInputs.ClusterDomain, false) + if err != nil { + return ci, nil, extensions.Degradedf(operatorv1.ResourceReadError, "failed to get KeyValidator config: %s", err) + } + } + + // Under certificate management, the query server needs its own keypair so it can run + // with different permissions than the apiserver. + var queryServerTLS certificatemanagement.KeyPairInterface + if in.CertificateManagement != nil { + queryServerTLS, err = ci.CertificateManager.GetOrCreateKeyPair( + ci.Client, + "query-server-tls", + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, ci.RenderInputs.ClusterDomain), + ) + if err != nil { + return ci, nil, fmt.Errorf("unable to get or create query server tls key pair: %w", err) + } + } + + // Modifiers run with no ImageSet, so resolve the images they need here. The query + // server and L7 admission controller containers run the combined calico image; the + // sidecar images are only needed when sidecar injection is enabled. + imageSet, err := imageset.GetImageSet(ctx, ci.Client, in.Variant) + if err != nil { + return ci, nil, err + } + calicoImage, err := components.GetReference(components.CombinedCalicoImage(in), in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return ci, nil, err + } + + var l7EnvoyImage, dikastesImage string + if applicationLayer != nil && + applicationLayer.Spec.SidecarInjection != nil && + *applicationLayer.Spec.SidecarInjection == operatorv1.SidecarEnabled { + l7EnvoyImage, err = components.GetReference(components.ComponentEnvoyProxy, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return ci, nil, err + } + dikastesImage, err = components.GetReference(components.ComponentDikastes, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return ci, nil, err + } + } + + // On a multi-tenant management cluster, each tenant's calico-apiserver ServiceAccount is + // granted Linseed access via a single ClusterRoleBinding with one subject per tenant + // namespace. Zero/single-tenant clusters leave this empty - the calico-system API server + // is covered by its own binding. + var bindingNamespaces []string + if e.opts.MultiTenant { + bindingNamespaces, err = utils.TenantNamespaces(ctx, ci.Client, nil) + if err != nil { + return ci, nil, fmt.Errorf("error reading tenant namespaces: %w", err) + } + } + + ci.RenderInputs.TrustedBundle = trustedBundle + ci.RenderInputs.Extension = apiServerRenderData{ + managementCluster: managementCluster, + managementClusterConnection: managementClusterConnection, + applicationLayer: applicationLayer, + queryServerTLS: queryServerTLS, + keyValidatorConfig: keyValidatorConfig, + l7EnvoyImage: l7EnvoyImage, + dikastesImage: dikastesImage, + calicoImage: calicoImage, + bindingNamespaces: bindingNamespaces, + cloud: e.opts.Cloud, + rbacManagementEnabled: rbacManagementEnabled, + } + return ci, nil, nil +} + +// CalicoCleanup deletes the Enterprise API server objects a prior Enterprise +// installation left behind. +type CalicoCleanup struct{} + +var _ extensions.APIServerExtension = CalicoCleanup{} + +func (CalicoCleanup) ExtendInputs(_ context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) { + return ci, nil, nil +} + +func (CalicoCleanup) Watches(ctrlruntime.Controller) error { + return nil +} + +func (CalicoCleanup) Modify(c render.Component, ri render.Inputs) render.Component { + t, ok := c.(render.APIServerComponent) + if !ok { + return c + } + + return extensions.Decorate(c, ri, operatorv1.Calico, func(create, del []client.Object) ([]client.Object, []client.Object) { + return cleanupAPIServer(ri, t.APIServerConfig(), create, del) + }) +} + +// modifyAPIServer layers Calico Enterprise behavior onto the rendered API server objects: +// the query server container and its volumes, audit logging on the aggregation API server +// container, the Enterprise RBAC objects, and the query server port on the Service. +func modifyAPIServer(ri render.Inputs, cfg *render.APIServerConfiguration, create, del []client.Object) ([]client.Object, []client.Object) { + data := apiServerData(ri) + c := &apiServer{cfg: cfg, calicoImage: data.calicoImage, data: data} + + // Ensure the deployment and its supporting objects exist. The base renders them when + // running an aggregation API server; in v3-CRD mode it queues them for deletion, but + // Enterprise always runs a query server, so render the skeleton ourselves and pull + // those objects back out of the delete list. + create, del = c.ensureDeployment(create, del) + + if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + c.layerDeployment(dep) + } + if svc, ok := extensions.FindObject[*corev1.Service](create, render.APIServerServiceName); ok { + c.addServicePorts(svc) + } + // Enterprise serves staged policies through the tiered-policy passthrough role. + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](create, render.TieredPolicyPassthruClusterRoleName); ok { + for i := range role.Rules { + if slices.Contains(role.Rules[i].Resources, "networkpolicies") { + role.Rules[i].Resources = append(role.Rules[i].Resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") + } + } + } + + // The L7 sidecar mutating webhook is driven by ApplicationLayer. The base always + // queues it for deletion; when sidecar injection is on, render it and pull it back + // out of the delete list. + if c.isSidecarInjectionEnabled() { + create = append(create, c.sidecarMutatingWebhookConfig()) + del = removeByRef(del, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) + } + + // Global Enterprise RBAC. + create = append(create, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) + if c.cfg.MultiTenant { + // Grant each tenant's calico-apiserver identity Linseed access. Bound across every tenant + // namespace via one ClusterRoleBinding (see linseedAccessClusterRoleBinding). + create = append(create, c.linseedAccessClusterRole(), c.linseedAccessClusterRoleBinding()) + } else { + // These resources are only installed in zero-tenant clusters. + create = append(create, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) + del = append(del, c.linseedAccessClusterRoleBinding(), c.linseedAccessClusterRole()) + } + if c.data.managementCluster != nil { + create = append(create, c.managedClusterWatchClusterRole()) + if c.cfg.MultiTenant { + create = append(create, c.multiTenantSecretsRBAC()...) + create = append(create, c.multiTenantManagedClusterAccessClusterRoles()...) + } else { + create = append(create, c.secretsRBAC()...) + } + } else { + // If we're not a management cluster, the API server doesn't need permissions to access secrets. + del = append(del, c.multiTenantSecretsRBAC()...) + del = append(del, c.secretsRBAC()...) + del = append(del, c.multiTenantManagedClusterAccessClusterRoles()...) + del = append(del, c.managedClusterWatchClusterRole()) + } + + // Namespaced Enterprise objects. + if c.cfg.TrustedBundle != nil { + create = append(create, c.cfg.TrustedBundle.ConfigMap(render.QueryserverNamespace)) + } + if c.data.managementClusterConnection != nil { + create = append(create, c.externalLinseedRoleBinding()) + } + + // Objects that only exist alongside the aggregation API server. + aggregationObjects := []client.Object{ + c.uiSettingsGroupGetterClusterRole(), + c.kubeControllerManagerUISettingsGroupGetterClusterRoleBinding(), + c.uiSettingsPassthruClusterRole(), + c.uiSettingsPassthruClusterRolebinding(), + c.auditPolicyConfigMap(), + } + if c.cfg.RequiresAggregationServer { + create = append(create, aggregationObjects...) + } else { + del = append(del, aggregationObjects...) + } + + // Clean up cluster-scoped resources that were created with the 'tigera' prefix. + if !c.cfg.HoldAPIServiceCutover { + del = append(del, c.deprecatedResources()...) + } + + // Re-apply deployment overrides so the modifier-added query server container picks up + // any per-container overrides. The override appliers use replace/merge semantics, so + // re-running over the render-applied containers is idempotent. + if dep, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + if overrides := c.cfg.APIServer.APIServerDeployment; overrides != nil { + rcomp.ApplyDeploymentOverrides(dep, overrides) + } + } + + return create, del +} + +// cleanupAPIServer deletes the Enterprise API server objects when running Calico, so a +// cluster switched from Enterprise to Calico does not leave them behind. +func cleanupAPIServer(ri render.Inputs, cfg *render.APIServerConfiguration, create, del []client.Object) ([]client.Object, []client.Object) { + c := &apiServer{cfg: cfg} + + del = append(del, c.tigeraAPIServerClusterRole(), c.tigeraAPIServerClusterRoleBinding()) + del = append(del, c.linseedAccessClusterRoleBinding(), c.linseedAccessClusterRole()) + if !c.cfg.MultiTenant { + del = append(del, c.tigeraUserClusterRole(), c.tigeraNetworkAdminClusterRole()) + } + del = append(del, c.multiTenantSecretsRBAC()...) + del = append(del, c.secretsRBAC()...) + del = append(del, c.multiTenantManagedClusterAccessClusterRoles()...) + del = append(del, c.managedClusterWatchClusterRole()) + + return create, del +} + +// layerDeployment adds the Enterprise additions to the rendered API server deployment: +// the query server container (and, under certificate management, its init container and +// volume), audit logging and the management-cluster tunnel args on the aggregation API +// server container, the L7 admission controller sidecar, and the Linseed token and +// trusted bundle volumes. +func (c *apiServer) layerDeployment(d *appsv1.Deployment) { + spec := &d.Spec.Template.Spec + if d.Spec.Template.Annotations == nil { + d.Spec.Template.Annotations = map[string]string{} + } + + // Audit logging and the management-cluster tunnel args are layered onto the + // aggregation API server container, which is only present when that server runs. + if c.cfg.RequiresAggregationServer { + { + ctr := render.MustContainer(spec, render.APIServerContainerName) + ctr.VolumeMounts = append(ctr.VolumeMounts, + corev1.VolumeMount{Name: auditLogsVolumeName, MountPath: "/var/log/calico/audit"}, + corev1.VolumeMount{Name: auditPolicyVolumeName, MountPath: "/etc/tigera/audit"}, + ) + ctr.Args = append(ctr.Args, + "--audit-policy-file=/etc/tigera/audit/policy.conf", + "--audit-log-path=/var/log/calico/audit/tsee-audit.log", + ) + ctr.Args = append(ctr.Args, c.managementClusterArgs()...) + // In case of OpenShift, apiserver needs privileged access to write audit logs to the + // host path volume. Audit logs are owned by root on hosts so we need to be root. + ctr.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) + } + + spec.Volumes = append(spec.Volumes, c.auditVolumes()...) + } + + spec.Containers = append(spec.Containers, c.queryServerContainer()) + if c.isSidecarInjectionEnabled() { + spec.Containers = append(spec.Containers, c.l7AdmissionControllerContainer()) + } + + // Under certificate management the query server gets its own cert init container and + // volume, since apiserver and queryserver may run with different UID:GID. + if c.data.queryServerTLS != nil { + init := c.data.queryServerTLS.InitContainer(render.APIServerNamespace, securitycontext.NewNonRootContext()) + spec.InitContainers = append(spec.InitContainers, init) + spec.Volumes = append(spec.Volumes, c.data.queryServerTLS.Volume()) + d.Spec.Template.Annotations[c.data.queryServerTLS.HashAnnotationKey()] = c.data.queryServerTLS.HashAnnotationValue() + } + + if c.data.managementClusterConnection != nil { + // Optional: the Secret is delivered over the Guardian tunnel, which can't be + // established until calico-apiserver is Ready. + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: render.LinseedTokenVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: fmt.Sprintf(render.LinseedTokenSecret, "calico-apiserver"), + Items: []corev1.KeyToPath{{Key: render.LinseedTokenKey, Path: render.LinseedTokenSubPath}}, + Optional: ptr.To(true), + }, + }, + }) + } + + if c.cfg.TrustedBundle != nil { + spec.Volumes = append(spec.Volumes, c.cfg.TrustedBundle.Volume()) + for k, v := range c.cfg.TrustedBundle.HashAnnotations() { + d.Spec.Template.Annotations[k] = v + } + } +} + +// managementClusterArgs returns the aggregation API server tunnel args for a management +// cluster, or nil when this isn't one. +func (c *apiServer) managementClusterArgs() []string { + mc := c.data.managementCluster + if mc == nil { + return nil + } + args := []string{"--enable-managed-clusters-create-api=true"} + if mc.Spec.Address != "" { + args = append(args, fmt.Sprintf("--managementClusterAddr=%s", mc.Spec.Address)) + } + if mc.Spec.TLS != nil && mc.Spec.TLS.SecretName != "" { + if mc.Spec.TLS.SecretName == render.ManagerTLSSecretName { + args = append(args, "--managementClusterCAType=Public") + } + args = append(args, fmt.Sprintf("--tunnelSecretName=%s", mc.Spec.TLS.SecretName)) + } + return args +} + +// ensureDeployment makes sure the API server Deployment and its supporting objects are +// in the create list. The base renders them when running an aggregation API server; when +// it doesn't (v3-CRD mode), it queues them for deletion, so render the skeleton and pull +// those objects back out of the delete list. +func (c *apiServer) ensureDeployment(create, del []client.Object) ([]client.Object, []client.Object) { + if _, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName); ok { + return create, del + } + skeleton := render.APIServerDeploymentObjects(c.cfg, c.calicoImage) + create = append(create, skeleton...) + for _, obj := range render.APIServerDeploymentObjectMeta() { + del = removeByRef(del, obj) + } + return create, del +} + +// removeByRef returns del with any object matching ref's kind, namespace, and name +// removed. +func removeByRef(del []client.Object, ref client.Object) []client.Object { + out := del[:0:0] + for _, o := range del { + if reflect.TypeOf(o) == reflect.TypeOf(ref) && + o.GetNamespace() == ref.GetNamespace() && + o.GetName() == ref.GetName() { + continue + } + out = append(out, o) + } + return out +} + +// addServicePorts adds the query server port and, when sidecar injection is enabled, the +// L7 admission controller port to the API server Service. +func (c *apiServer) addServicePorts(s *corev1.Service) { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName) + s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ + Name: render.QueryServerPortName, + Port: render.QueryServerPort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), + }) + if c.isSidecarInjectionEnabled() { + l7Port := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName) + s.Spec.Ports = append(s.Spec.Ports, corev1.ServicePort{ + Name: render.L7AdmissionControllerPortName, + Port: render.L7AdmissionControllerPort, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt32(l7Port.ContainerPort), + }) + } +} + +// l7AdmissionControllerContainer is the L7 admission controller sidecar, rendered when +// ApplicationLayer sidecar injection is enabled. +func (c *apiServer) l7AdmissionControllerContainer() corev1.Container { + volumeMounts := []corev1.VolumeMount{ + c.cfg.TLSKeyPair.VolumeMount(rmeta.OSTypeLinux), + } + + l7Port := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName).ContainerPort + + dataplane := "iptables" + if c.cfg.Installation.IsNftables() { + dataplane = "nftables" + } + + return corev1.Container{ + Name: render.L7AdmissionControllerContainerName, + Image: c.calicoImage, + Command: []string{components.CalicoBinaryPath, "component", "l7-admission-controller"}, + Env: []corev1.EnvVar{ + {Name: "L7ADMCTRL_TLSCERTPATH", Value: c.cfg.TLSKeyPair.VolumeMountCertificateFilePath()}, + {Name: "L7ADMCTRL_TLSKEYPATH", Value: c.cfg.TLSKeyPair.VolumeMountKeyFilePath()}, + {Name: "L7ADMCTRL_ENVOYIMAGE", Value: c.data.l7EnvoyImage}, + {Name: "L7ADMCTRL_DIKASTESIMAGE", Value: c.data.dikastesImage}, + {Name: "L7ADMCTRL_LISTENADDR", Value: fmt.Sprintf(":%d", l7Port)}, + {Name: "DATAPLANE", Value: dataplane}, + }, + VolumeMounts: volumeMounts, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/live", + Port: intstr.FromInt32(l7Port), + Scheme: corev1.URISchemeHTTPS, + }, + }, + }, + } +} + +// sidecarMutatingWebhookConfig is the L7 sidecar injection webhook, rendered when +// ApplicationLayer sidecar injection is enabled. +func (c *apiServer) sidecarMutatingWebhookConfig() *admregv1.MutatingWebhookConfiguration { + var cacert []byte + svcPort := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName).ContainerPort + + svcpath := "/sidecar-webhook" + svcref := admregv1.ServiceReference{ + Name: render.QueryserverServiceName, + Namespace: render.QueryserverNamespace, + Path: &svcpath, + Port: &svcPort, + } + failpol := admregv1.Fail + labelsel := metav1.LabelSelector{ + MatchLabels: map[string]string{ + "applicationlayer.projectcalico.org/sidecar": "true", + }, + } + rules := []admregv1.RuleWithOperations{ + { + Rule: admregv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + Operations: []admregv1.OperationType{admregv1.Create}, + }, + } + sidefx := admregv1.SideEffectClassNone + if !c.cfg.TLSKeyPair.UseCertificateManagement() { + cacert = c.cfg.TLSKeyPair.GetIssuer().GetCertificatePEM() + } else { + cacert = c.cfg.Installation.CertificateManagement.CACert + } + return &admregv1.MutatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "MutatingWebhookConfiguration", + APIVersion: "admissionregistration.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}, + Webhooks: []admregv1.MutatingWebhook{ + { + AdmissionReviewVersions: []string{"v1"}, + ClientConfig: admregv1.WebhookClientConfig{ + Service: &svcref, + CABundle: cacert, + }, + Name: "sidecar.projectcalico.org", + FailurePolicy: &failpol, + ObjectSelector: &labelsel, + Rules: rules, + SideEffects: &sidefx, + }, + }, + } +} + +// modifyAPIServerPolicy adds the enterprise additions to the API server network policy: +// the OIDC egress rule (when an OIDC key validator is configured) and the L7 admission +// controller ingress port (when sidecar injection is enabled). The base policy carries +// neither. +func modifyAPIServerPolicy(ri render.Inputs, cfg *render.APIServerConfiguration, create, del []client.Object) ([]client.Object, []client.Object) { + c := &apiServer{cfg: cfg, data: apiServerData(ri)} + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](create, render.APIServerPolicyName) + if !ok { + return create, del + } + + // Insert the OIDC egress rule before the trailing Pass rule so it is evaluated. + if c.data.keyValidatorConfig != nil { + if parsedURL, err := url.Parse(c.data.keyValidatorConfig.Issuer()); err == nil { + oidc := networkpolicy.GetOIDCEgressRule(parsedURL) + egress := policy.Spec.Egress + if n := len(egress); n > 0 && egress[n-1].Action == v3.Pass { + policy.Spec.Egress = append(egress[:n-1:n-1], oidc, egress[n-1]) + } else { + policy.Spec.Egress = append(egress, oidc) + } + } + } + + // Allow the kube-apiserver to reach the L7 admission controller. + if c.isSidecarInjectionEnabled() { + l7Port := render.GetContainerPort(c.cfg, render.L7AdmissionControllerContainerName).ContainerPort + for i := range policy.Spec.Ingress { + policy.Spec.Ingress[i].Destination.Ports = append(policy.Spec.Ingress[i].Destination.Ports, + numorstring.Port{MinPort: uint16(l7Port), MaxPort: uint16(l7Port)}) + } + } + + return create, del +} + +// auditVolumes are the host-path audit log and audit policy volumes used by the +// aggregation API server container. +func (c *apiServer) auditVolumes() []corev1.Volume { + return []corev1.Volume{ + { + Name: auditLogsVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/log/calico/audit", + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }, + }, + }, + { + Name: auditPolicyVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: auditPolicyVolumeName}, + Items: []corev1.KeyToPath{ + { + Key: "config", + Path: "policy.conf", + }, + }, + }, + }, + }, + } +} + +func (c *apiServer) multiTenantSecretsRBAC() []client.Object { + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.data.managementCluster, true) +} + +func (c *apiServer) secretsRBAC() []client.Object { + return render.TunnelSecretRBAC(render.APIServerSecretsRBACName, render.APIServerServiceAccountName, c.data.managementCluster, false) +} + +// linseedAccessClusterRole is a minimal, least-privilege ClusterRole granting the calico-apiserver +// identity read access to Linseed policy activity data (for queryserver enrichment). On a multi-tenant +// management cluster it is bound to each tenant's calico-apiserver ServiceAccount so that tenant's +// managed clusters can reach Linseed. The full backing-storage/queryserver rules live on the +// calico-apiserver ClusterRole, which is bound only to the calico-system API server itself. +// +// Calico Enterprise, multi-tenant only. +func (c *apiServer) linseedAccessClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: linseedAccessClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + // Read access to Linseed policy activity data for queryserver enrichment. + APIGroups: []string{"linseed.tigera.io"}, + Resources: []string{"policyactivity"}, + Verbs: []string{"get"}, + }, + }, + } +} + +// linseedAccessClusterRoleBinding binds the Linseed-access ClusterRole to the calico-apiserver +// ServiceAccount in each tenant namespace. Linseed authorizes with a cluster-scoped +// SubjectAccessReview, so this is a single ClusterRoleBinding with one ServiceAccount subject per +// tenant namespace - mirroring how compliance, intrusion-detection, and the manager grant their +// managed-cluster components Linseed access. +// +// Calico Enterprise, multi-tenant only. +func (c *apiServer) linseedAccessClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return rcomp.ClusterRoleBinding(linseedAccessClusterRoleName, linseedAccessClusterRoleName, render.APIServerServiceAccountName, c.data.bindingNamespaces) +} + +func (c *apiServer) queryServerContainer() corev1.Container { + queryServerTargetPort := render.GetContainerPort(c.cfg, render.TigeraAPIServerQueryServerContainerName).ContainerPort + + var tlsSecret certificatemanagement.KeyPairInterface + if c.data.queryServerTLS != nil { + tlsSecret = c.data.queryServerTLS + } else { + tlsSecret = c.cfg.TLSKeyPair + } + env := []corev1.EnvVar{ + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", queryServerTargetPort)}, + {Name: "TLS_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, + {Name: "TLS_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, + } + if c.cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "TRUSTED_BUNDLE_PATH", Value: c.cfg.TrustedBundle.MountPath()}) + } + + if render.HostNetwork(c.cfg) { + env = append(env, c.cfg.K8SServiceEndpoint.EnvVars()...) + } else { + env = append(env, c.cfg.K8SServiceEndpointPodNetwork.EnvVars()...) + } + + if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if c.data.keyValidatorConfig != nil { + env = append(env, c.data.keyValidatorConfig.RequiredEnv("")...) + } + + linseedURL := relasticsearch.LinseedEndpoint(rmeta.OSTypeLinux, c.cfg.ClusterDomain, render.ElasticsearchNamespace, c.data.managementClusterConnection != nil, false) + env = append(env, + corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, + corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, + corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, + ) + if c.data.managementClusterConnection != nil { + env = append(env, + corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, + corev1.EnvVar{Name: "LINSEED_TOKEN", Value: render.GetLinseedTokenPath(true)}, + ) + } + if c.cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "LINSEED_CA", Value: c.cfg.TrustedBundle.MountPath()}) + } + + // set LogLEVEL for queryserver container + if logging := c.cfg.APIServer.Logging; logging != nil && + logging.QueryServerLogging != nil && logging.QueryServerLogging.LogSeverity != nil { + env = append(env, + corev1.EnvVar{Name: "LOGLEVEL", Value: strings.ToLower(string(*logging.QueryServerLogging.LogSeverity))}) + } else { + // set default LOGLEVEL to info when not set by the user + env = append(env, corev1.EnvVar{Name: "LOGLEVEL", Value: "info"}) + } + + volumeMounts := []corev1.VolumeMount{ + tlsSecret.VolumeMount(rmeta.OSTypeLinux), + } + if c.cfg.TrustedBundle != nil { + volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(rmeta.OSTypeLinux)...) + } + if c.data.managementClusterConnection != nil { + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: render.LinseedTokenVolumeName, + MountPath: render.LinseedVolumeMountPath, + }) + } + + container := corev1.Container{ + Name: render.TigeraAPIServerQueryServerContainerName, + Image: c.calicoImage, + Command: []string{components.CalicoBinaryPath, "component", "queryserver"}, + Env: env, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/version", + Port: intstr.FromInt32(queryServerTargetPort), + Scheme: corev1.URISchemeHTTPS, + }, + }, + InitialDelaySeconds: 90, + }, + SecurityContext: securitycontext.NewNonRootContext(), + VolumeMounts: volumeMounts, + } + return container +} + +func (c *apiServer) externalLinseedRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-linseed", + Namespace: render.APIServerNamespace, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: render.TigeraLinseedSecretsClusterRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.GuardianServiceAccountName, + Namespace: render.GuardianNamespace, + }, + }, + } +} + +func (c *apiServer) tigeraAPIServerClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + { + // Read access to Linseed policy activity data for queryserver enrichment. + APIGroups: []string{"linseed.tigera.io"}, + Resources: []string{"policyactivity"}, + Verbs: []string{"get"}, + }, + { + // Calico Enterprise backing storage. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "bfdconfigurations", + "deeppacketinspections", + "deeppacketinspections/status", + "egressgatewaypolicies", + "externalnetworks", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalreports", + "globalreports/status", + "globalreporttypes", + "globalthreatfeeds", + "globalthreatfeeds/status", + "licensekeys", + "managedclusters", + "managedclusters/status", + "networks", + "packetcaptures", + "packetcaptures/status", + "policyrecommendationscopes", + "policyrecommendationscopes/status", + "remoteclusterconfigurations", + "securityeventwebhooks", + "securityeventwebhooks/status", + "uisettings", + "uisettingsgroups", + }, + Verbs: []string{ + "get", + "list", + "watch", + "create", + "update", + "delete", + "patch", + }, + }, + { + // The queryserver's RBAC calculator needs to list tiers, + // uisettingsgroups, and managedclusters via the aggregated + // API to evaluate user permissions for the /policies endpoint. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "tiers", + "uisettingsgroups", + "managedclusters", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Required by the AuthorizationReview calculator in queryserver to evaluate + // RBAC permissions for users. + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{ + "clusterroles", + "clusterrolebindings", + "roles", + "rolebindings", + }, + Verbs: []string{"get", "list", "watch"}, + }, + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.APIServerName, + }, + Rules: rules, + } +} + +func (c *apiServer) tigeraAPIServerClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.APIServerName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.APIServerServiceAccountName, + Namespace: render.APIServerNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: render.APIServerName, + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func (c *apiServer) uiSettingsGroupGetterClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettingsgroup-getter", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "uisettingsgroups", + }, + Verbs: []string{"get"}, + }, + }, + } +} + +func (c *apiServer) kubeControllerManagerUISettingsGroupGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettingsgroup-getter", + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-uisettingsgroup-getter", + APIGroup: "rbac.authorization.k8s.io", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "User", + Name: "system:kube-controller-manager", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + } +} + +func (c *apiServer) tigeraUserClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + // List requests that the Tigera manager needs. + { + APIGroups: []string{ + "projectcalico.org", + "networking.k8s.io", + "extensions", + "", + }, + // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior + // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for + // more details). Similar for all tiered policy resource types. + Resources: []string{ + "tiers", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "namespaces", + "globalnetworksets", + "networksets", + "managedclusters", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedkubernetesnetworkpolicies", + "policyrecommendationscopes", + }, + Verbs: []string{"watch", "list"}, + }, + { + APIGroups: []string{"policy.networking.k8s.io"}, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures/files"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to view Networks. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Additional "list" requests required to view flows. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Additional "list" requests required to view serviceaccount labels. + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + // Access for WAF API to read in coreruleset configmap + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"coreruleset-default"}, + Verbs: []string{"get"}, + }, + // Access to statistics. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "https:calico-api:8080", "calico-node-prometheus:9090", + }, + Verbs: []string{"get", "create"}, + }, + // Access to policies in all tiers + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"get"}, + }, + // List and download the reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list"}, + }, + // Access to hostendpoints from the UI ServiceGraph. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"get", "list"}, + }, + // List and view the threat defense configuration + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalthreatfeeds", + "globalthreatfeeds/status", + "securityeventwebhooks", + }, + Verbs: []string{"get", "watch", "list"}, + }, + } + + // User can: + // - read UISettings in the cluster-settings group (not on Calico Cloud, which only exposes + // per-user UISettings) + // - read and write UISettings in the user-settings group + // Default settings group and settings are created in manager.go. + if c.data.cloud { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"user-settings"}, + }, + ) + } else { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"get", "list", "watch"}, + ResourceNames: []string{"cluster-settings"}, + }, + ) + } + + rules = append(rules, []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"user-settings"}, + }, + // Allow the user to read applicationlayers to detect if WAF is enabled/disabled. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, + Verbs: []string{"get"}, + }, + // Allow the user to read the gatewayapis CR to detect if Gateway API is enabled/disabled. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"gatewayapis"}, + Verbs: []string{"get"}, + }, + // Allow the user to read Gateways and HTTPRoutes to offer as WAF policy attach targets. + { + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways", "httproutes"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to view WAF policies, plugins, and validation policies. + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "globalwafpolicies", + "globalwafplugins", + "globalwafvalidationpolicies", + "wafpolicies", + "wafplugins", + "wafvalidationpolicies", + }, + Verbs: []string{"get", "watch", "list"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to read services to view WAF configuration. + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list"}, + }, + // Allow the user to only view securityeventwebhooks. + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list"}, + }, + }...) + + // Privileges for lma.tigera.io have no effect on managed clusters. + if c.data.managementClusterConnection == nil { + // Access to flow logs, audit logs, and statistics, plus logging into Kibana for oidc users. + // Calico Cloud also gets runtime logs. + resourceNames := []string{"flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations"} + if c.data.cloud { + resourceNames = append([]string{"runtime"}, resourceNames...) + } + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: resourceNames, + Verbs: []string{"get"}, + }) + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-ui-user", + }, + Rules: rules, + } +} + +func (c *apiServer) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { + rules := []rbacv1.PolicyRule{ + // Full access to all network policies + { + APIGroups: []string{ + "projectcalico.org", + "networking.k8s.io", + "extensions", + }, + // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior + // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for + // more details). Similar for all tiered policy resource types. + Resources: []string{ + "tiers", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedkubernetesnetworkpolicies", + "globalnetworksets", + "networksets", + "managedclusters", + "packetcaptures", + "policyrecommendationscopes", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + { + APIGroups: []string{ + "policy.networking.k8s.io", + }, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"packetcaptures/files"}, + Verbs: []string{"get", "delete"}, + }, + // Allow the user to CRUD Networks. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + // Additional "list" requests that the Tigera Secure manager needs + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"watch", "list"}, + }, + // Additional "list" requests required to view flows. + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Additional "list" requests required to view serviceaccount labels. + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + // Access for WAF API to read in coreruleset configmap + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"coreruleset-default"}, + Verbs: []string{"get"}, + }, + // Access to statistics. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "https:calico-api:8080", "calico-node-prometheus:9090", + }, + Verbs: []string{"get", "create"}, + }, + // Manage globalreport configuration, view report generation status, and list reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports"}, + Verbs: []string{"*"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreports/status"}, + Verbs: []string{"get", "list", "watch"}, + }, + // List and download the reports in the Tigera Secure manager. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes"}, + Verbs: []string{"get"}, + }, + // Access to cluster information containing Calico and EE versions from the UI. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list"}, + }, + // Access to hostendpoints from the UI ServiceGraph. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"get", "list"}, + }, + // Manage the threat defense configuration + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "alertexceptions", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "globalthreatfeeds", + "globalthreatfeeds/status", + "securityeventwebhooks", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + } + + // User can: + // - read and write UISettings in the cluster-settings group, and rename the group (not on Calico + // Cloud, which only exposes per-user UISettings) + // - read and write UISettings in the user-settings group, and rename the group + // Default settings group and settings are created in manager.go. + if c.data.cloud { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"user-settings"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"user-settings"}, + }, + ) + } else { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get", "patch", "update"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + ) + } + + rules = append(rules, []rbacv1.PolicyRule{ + // Allow the user to read and write applicationlayers to enable/disable WAF. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, + Verbs: []string{"get", "update", "patch", "create", "delete"}, + }, + // Allow the user to read the gatewayapis CR to detect if Gateway API is enabled/disabled. + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"gatewayapis"}, + Verbs: []string{"get"}, + }, + // Allow the user to read Gateways and HTTPRoutes to offer as WAF policy attach targets. + { + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways", "httproutes"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Allow the user to manage WAF policies, plugins, and validation policies. + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "globalwafpolicies", + "globalwafplugins", + "globalwafvalidationpolicies", + "wafpolicies", + "wafplugins", + "wafvalidationpolicies", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + }, + // Allow the user to read deployments to view WAF configuration. + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "watch", "patch"}, + }, + // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list"}, + }, + // Allow the user to perform CRUD operations on securityeventwebhooks. + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, + }, + // Allow the user to create secrets. + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + }, + Verbs: []string{"create"}, + }, + // Allow the user to patch webhooks-secret secret. + { + APIGroups: []string{""}, + Resources: []string{ + "secrets", + }, + ResourceNames: []string{ + "webhooks-secret", + }, + Verbs: []string{"patch"}, + }, + }...) + + // ui-apis writes these impersonating the caller, so the apiserver enforces escalation + // against the user's own permissions. + if c.data.rbacManagementEnabled { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "roles"}, + Verbs: []string{"get", "list", "watch"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterrolebindings", "rolebindings"}, + Verbs: []string{"get", "list", "watch", "create", "update", "delete"}, + }, + ) + } + + // Privileges for lma.tigera.io have no effect on managed clusters. + if c.data.managementClusterConnection == nil { + // Access to flow logs, audit logs, and statistics, plus Elasticsearch superuser access once + // logged into Kibana. Calico Cloud also gets runtime logs. + resourceNames := []string{"flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations"} + if c.data.cloud { + resourceNames = append([]string{"runtime"}, resourceNames...) + } + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: resourceNames, + Verbs: []string{"get"}, + }) + } + + // In v3 CRD / webhooks mode there is no aggregated apiserver, and the + // calico-uisettings-passthrough ClusterRole that normally grants the broad + // uisettings permission isn't deployed. Grant write verbs here so the + // calico-webhooks UISettings handler (which narrows access via a SAR on + // uisettingsgroups/data) gets invoked instead of being short-circuited by + // kube-apiserver RBAC. + if !c.cfg.RequiresAggregationServer { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings"}, + Verbs: []string{"create", "update", "delete", "patch"}, + }) + } + + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-network-admin", + }, + Rules: rules, + } +} + +func (c *apiServer) uiSettingsPassthruClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettings-passthrough", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings"}, + Verbs: []string{"*"}, + }, + }, + } +} + +func (c *apiServer) uiSettingsPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-uisettings-passthrough", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "Group", + Name: "system:authenticated", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: "calico-uisettings-passthrough", + APIGroup: "rbac.authorization.k8s.io", + }, + } +} + +func (c *apiServer) auditPolicyConfigMap() *corev1.ConfigMap { + const defaultAuditPolicy = `apiVersion: audit.k8s.io/v1 +kind: Policy +rules: +- level: RequestResponse + omitStages: + - RequestReceived + verbs: + - create + - patch + - update + - delete + resources: + - group: projectcalico.org + resources: + - globalnetworkpolicies + - networkpolicies + - stagedglobalnetworkpolicies + - stagednetworkpolicies + - stagedkubernetesnetworkpolicies + - globalnetworksets + - networksets + - tiers + - hostendpoints` + + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + // This object is for Enterprise only, so pass it explicitly. + Namespace: render.APIServerNamespace, + Name: auditPolicyVolumeName, + }, + Data: map[string]string{ + "config": defaultAuditPolicy, + }, + } +} + +func (c *apiServer) multiTenantManagedClusterAccessClusterRoles() []client.Object { + var objects []client.Object + objects = append(objects, &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.MultiTenantManagedClustersAccessClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"managedclusters"}, + Verbs: []string{ + // The Authentication Proxy in Voltron checks if Enterprise Components (using impersonation headers for + // the service in the canonical namespace) can get a managed clusters before sending the request down the tunnel. + // This ClusterRole will be assigned to each component using a RoleBinding in the canonical or tenant namespace. + "get", + }, + }, + }, + }) + + return objects +} + +func (c *apiServer) managedClusterWatchClusterRole() client.Object { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"managedclusters"}, + Verbs: []string{ + "get", "list", "watch", + }, + }, + }, + } +} + +func (c *apiServer) deprecatedResources() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, + }, + + // delegateAuthClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-delegate-auth"}, + }, + + // authClusterRole + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, + }, + + // authClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, + }, + // authReaderRoleBinding - need clean up in diff namespace kube-system + &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-auth-reader", + Namespace: "kube-system", + }, + }, + // webhookReaderClusterRole + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-webhook-reader"}, + }, + + // webhookReaderClusterRoleBinding + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-webhook-reader"}, + }, + + // calico-apiserver CR and CRB + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, + }, + + &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, + }, + &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, + }, + + // Clean up legacy secrets in the tigera-operator namespace + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-api-cert", Namespace: common.OperatorNamespace()}, + }, + } +} diff --git a/pkg/enterprise/apiserver/extension_test.go b/pkg/enterprise/apiserver/extension_test.go new file mode 100644 index 0000000000..7db82e2c78 --- /dev/null +++ b/pkg/enterprise/apiserver/extension_test.go @@ -0,0 +1,838 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiserver_test + +import ( + "context" + "fmt" + "slices" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + admregv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/rbacmanagement" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +const apiServerClusterDomain = "cluster.local" + +// apiServerControllerInputs builds a controller inputs for the API server controller, +// seeded with a fake client that holds objs. The returned context carries a real +// certificate manager and trusted bundle, so ExtendInputs can create the query server +// cert and the bundle the modifiers consume. +func apiServerControllerInputs(variant operatorv1.ProductVariant, install *operatorv1.InstallationSpec, objs ...client.Object) controller.Inputs { + return apiServerControllerInputsWith(ctrlrfake.DefaultFakeClientBuilder(apiServerScheme()).Build(), variant, install, objs...) +} + +// apiServerControllerInputsWith is apiServerControllerInputs against a caller-supplied client. +func apiServerControllerInputsWith(c client.WithWatch, variant operatorv1.ProductVariant, install *operatorv1.InstallationSpec, objs ...client.Object) controller.Inputs { + for _, o := range objs { + Expect(c.Create(context.Background(), o)).NotTo(HaveOccurred()) + } + + if install == nil { + install = &operatorv1.InstallationSpec{Variant: variant} + } + + certManager, err := certificatemanager.Create(c, install, apiServerClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: install, + ClusterDomain: apiServerClusterDomain, + TrustedBundle: certManager.CreateTrustedBundle(), + }, + Client: c, + CertificateManager: certManager, + } +} + +func apiServerScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + return scheme +} + +// failingConfigMapInputs returns controller inputs whose client fails every read of +// the named ConfigMap with readErr. +func failingConfigMapInputs(name string, readErr error) controller.Inputs { + c := ctrlrfake.DefaultFakeClientBuilder(apiServerScheme()).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.ConfigMap); ok && key.Name == name { + return readErr + } + return c.Get(ctx, key, obj, opts...) + }, + }).Build() + return apiServerControllerInputsWith(c, operatorv1.CalicoEnterprise, nil) +} + +// rbacManagementGate builds the admin-owned ConfigMap that switches the RBAC +// management UI on for a cluster. +func rbacManagementGate(enabled string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: rbacmanagement.ConfigMapName, Namespace: common.CalicoNamespace}, + Data: map[string]string{rbacmanagement.ConfigMapKey: enabled}, + } +} + +// apiServerKeyPair issues the API server TLS keypair from the inputs' certificate +// manager, the way the controller does before rendering. +func apiServerKeyPair(ci controller.Inputs) certificatemanagement.KeyPairInterface { + dnsNames := dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, ci.RenderInputs.ClusterDomain) + kp, err := ci.CertificateManager.GetOrCreateKeyPair(ci.Client, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) + Expect(err).NotTo(HaveOccurred()) + return kp +} + +var _ = Describe("API server enterprise controller extension", func() { + managementCluster := func() *operatorv1.ManagementCluster { + return &operatorv1.ManagementCluster{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ManagementClusterSpec{ + Address: "example.com:1234", + TLS: &operatorv1.TLS{SecretName: render.VoltronTunnelSecretName}, + }, + } + } + + tunnelSecret := func() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: render.VoltronTunnelSecretName, Namespace: common.OperatorNamespace()}, + Data: map[string][]byte{"cert": []byte("a"), "key": []byte("b")}, + } + } + + managementClusterConnection := func() *operatorv1.ManagementClusterConnection { + return &operatorv1.ManagementClusterConnection{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + } + } + + Describe("configuration", func() { + extendInputs := func(objs ...client.Object) error { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil, objs...) + _, _, err := ext.APIServer().ExtendInputs(ctx, ci) + return err + } + + It("accepts a cluster with neither a ManagementCluster nor a ManagementClusterConnection", func() { + Expect(extendInputs()).NotTo(HaveOccurred()) + }) + + It("accepts a management cluster", func() { + Expect(extendInputs(managementCluster(), tunnelSecret())).NotTo(HaveOccurred()) + }) + + It("accepts a managed cluster", func() { + Expect(extendInputs(managementClusterConnection())).NotTo(HaveOccurred()) + }) + + It("rejects a cluster that is both a management cluster and a managed cluster", func() { + err := extendInputs(managementCluster(), tunnelSecret(), managementClusterConnection()) + reason, ok := extensions.DegradedReason(err) + Expect(ok).To(BeTrue()) + Expect(reason).To(Equal(operatorv1.ResourceValidationError)) + }) + }) + + Describe("Dex", func() { + readyAuthentication := func() *operatorv1.Authentication { + return &operatorv1.Authentication{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Status: operatorv1.AuthenticationStatus{State: operatorv1.TigeraStatusReady}, + } + } + + It("reports not ready while the Dex TLS secret is missing", func() { + // WithObjects, since Create drops the status the extension keys off. + c := ctrlrfake.DefaultFakeClientBuilder(apiServerScheme()).WithObjects(readyAuthentication()).Build() + ci := apiServerControllerInputsWith(c, operatorv1.CalicoEnterprise, nil) + + _, _, err := ext.APIServer().ExtendInputs(ctx, ci) + reason, ok := extensions.DegradedReason(err) + Expect(ok).To(BeTrue()) + Expect(reason).To(Equal(operatorv1.ResourceNotReady)) + Expect(err.Error()).To(ContainSubstring(render.DexTLSSecretName)) + }) + }) +}) + +var _ = Describe("API server enterprise modifier", func() { + // renderAPIServerWith builds the base API server objects and runs the extension + // over them, returning the create and delete lists it produced. + renderAPIServerWith := func(s extensions.Extensions, ci controller.Inputs, ri render.Inputs, kp certificatemanagement.KeyPairInterface) ([]client.Object, []client.Object) { + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: true, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: ci.RenderInputs.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: kp, + TrustedBundle: ri.TrustedBundle, + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + + return s.APIServer().Modify(extensionstest.APIServerStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: cfg}, ri).Objects() + } + + renderAPIServer := func(ci controller.Inputs, ri render.Inputs, kp certificatemanagement.KeyPairInterface) ([]client.Object, []client.Object) { + return renderAPIServerWith(ext, ci, ri, kp) + } + + apiServerDeployment := func(objs []client.Object) *appsv1.Deployment { + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, render.APIServerName) + Expect(ok).To(BeTrue()) + return dp + } + + container := func(dp *appsv1.Deployment, name string) *corev1.Container { + for i := range dp.Spec.Template.Spec.Containers { + if dp.Spec.Template.Spec.Containers[i].Name == name { + return &dp.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + It("adds no enterprise objects when the operator runs as Calico", func() { + ci := apiServerControllerInputs(operatorv1.Calico, nil) + eci, _, err := calicoExt.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServerWith(calicoExt, ci, ri, apiServerKeyPair(ci)) + + // Only the cleanup is registered as Calico, and it only queues deletes. + _, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-ui-user") + Expect(ok).To(BeFalse()) + dp := apiServerDeployment(objs) + Expect(container(dp, render.TigeraAPIServerQueryServerContainerName)).To(BeNil()) + }) + + It("layers the query server, enterprise RBAC, audit policy, and query server port on", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(ci, ri, apiServerKeyPair(ci)) + + // The query server container is layered onto the deployment. + dp := apiServerDeployment(objs) + Expect(container(dp, render.TigeraAPIServerQueryServerContainerName)).NotTo(BeNil()) + + // Enterprise RBAC. + for _, name := range []string{"calico-apiserver", "tigera-ui-user", "tigera-network-admin", "calico-uisettingsgroup-getter", "calico-uisettings-passthrough"} { + _, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, name) + Expect(ok).To(BeTrue(), "expected ClusterRole %q", name) + } + + // The user and network-admin roles grant access to WAF policy resources. + uiUser, found := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-ui-user") + Expect(found).To(BeTrue()) + Expect(uiUser.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "globalwafpolicies", + "globalwafplugins", + "globalwafvalidationpolicies", + "wafpolicies", + "wafplugins", + "wafvalidationpolicies", + }, + Verbs: []string{"get", "watch", "list"}, + })) + + networkAdmin, found := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-network-admin") + Expect(found).To(BeTrue()) + Expect(networkAdmin.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "globalwafpolicies", + "globalwafplugins", + "globalwafvalidationpolicies", + "wafpolicies", + "wafplugins", + "wafvalidationpolicies", + }, + Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, + })) + + // Both roles can read the Gateway API, so the WAF UI can detect it and offer + // Gateways and HTTPRoutes as policy attach targets. + for _, role := range []*rbacv1.ClusterRole{uiUser, networkAdmin} { + Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"gatewayapis"}, + Verbs: []string{"get"}, + })) + Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways", "httproutes"}, + Verbs: []string{"get", "list", "watch"}, + })) + } + + // Audit policy configmap. + _, ok := extensions.FindObject[*corev1.ConfigMap](objs, "calico-audit-policy") + Expect(ok).To(BeTrue()) + + // The query server port is added to the Service. + svc, ok := extensions.FindObject[*corev1.Service](objs, render.APIServerServiceName) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(ContainElement(HaveField("Name", render.QueryServerPortName))) + }) + + DescribeTable("grants tigera-network-admin the role management verbs only when RBAC management is enabled", + func(gate *corev1.ConfigMap, expected bool) { + var objs []client.Object + if gate != nil { + objs = append(objs, gate) + } + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil, objs...) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + Expect(err).NotTo(HaveOccurred()) + + rendered, _ := renderAPIServer(ci, eci.RenderInputs, apiServerKeyPair(ci)) + networkAdmin, ok := extensions.FindObject[*rbacv1.ClusterRole](rendered, "tigera-network-admin") + Expect(ok).To(BeTrue()) + + // ui-apis writes bindings impersonating the caller, so the caller's own role + // has to carry the verbs or the apiserver's escalation check rejects it. + matcher := ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterrolebindings", "rolebindings"}, + Verbs: []string{"get", "list", "watch", "create", "update", "delete"}, + }) + if expected { + Expect(networkAdmin.Rules).To(matcher) + } else { + Expect(networkAdmin.Rules).NotTo(matcher) + } + }, + Entry("enabled", rbacManagementGate("true"), true), + Entry("disabled", rbacManagementGate("false"), false), + Entry("no ConfigMap", nil, false), + ) + + It("reads the gate on a managed cluster, which carries tigera-network-admin too", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil, + &operatorv1.ManagementClusterConnection{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}}, + rbacManagementGate("true")) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(ci, eci.RenderInputs, apiServerKeyPair(ci)) + networkAdmin, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-network-admin") + Expect(ok).To(BeTrue()) + Expect(networkAdmin.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterrolebindings", "rolebindings"}, + Verbs: []string{"get", "list", "watch", "create", "update", "delete"}, + })) + }) + + It("fails the reconcile when the gate ConfigMap cannot be read", func() { + // Unknown state is not the same as absent, so it must not render as disabled. + readErr := fmt.Errorf("the API server is having a bad day") + ci := failingConfigMapInputs(rbacmanagement.ConfigMapName, readErr) + + _, _, err := ext.APIServer().ExtendInputs(ctx, ci) + Expect(err).To(MatchError(readErr)) + }) + + Context("Calico Cloud", func() { + cloudExt := func() extensions.Extensions { + return enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{Cloud: true}) + } + + uiSettingsRules := func(role *rbacv1.ClusterRole) []rbacv1.PolicyRule { + var rules []rbacv1.PolicyRule + for _, r := range role.Rules { + if slices.Contains(r.Resources, "uisettingsgroups") || slices.Contains(r.Resources, "uisettingsgroups/data") { + rules = append(rules, r) + } + } + return rules + } + + lmaResourceNames := func(role *rbacv1.ClusterRole) []string { + for _, r := range role.Rules { + if slices.Contains(r.APIGroups, "lma.tigera.io") { + return r.ResourceNames + } + } + return nil + } + + objectsFor := func(s extensions.Extensions) ([]client.Object, []client.Object) { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil) + eci, _, err := s.APIServer().ExtendInputs(ctx, ci) + Expect(err).NotTo(HaveOccurred()) + return renderAPIServerWith(s, ci, eci.RenderInputs, apiServerKeyPair(ci)) + } + + It("exposes only the user-settings UISettings group", func() { + objs, _ := objectsFor(cloudExt()) + + uiUser, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-ui-user") + Expect(ok).To(BeTrue()) + Expect(uiSettingsRules(uiUser)).To(ConsistOf( + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"user-settings"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"user-settings"}, + }, + )) + + networkAdmin, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-network-admin") + Expect(ok).To(BeTrue()) + Expect(uiSettingsRules(networkAdmin)).To(ConsistOf( + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"user-settings"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"user-settings"}, + }, + )) + }) + + It("grants access to runtime logs", func() { + objs, _ := objectsFor(cloudExt()) + + uiUser, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-ui-user") + Expect(ok).To(BeTrue()) + Expect(lmaResourceNames(uiUser)).To(ContainElement("runtime")) + + networkAdmin, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-network-admin") + Expect(ok).To(BeTrue()) + Expect(lmaResourceNames(networkAdmin)).To(ContainElement("runtime")) + }) + + It("leaves the cluster-settings group and omits runtime logs off cloud", func() { + objs, _ := objectsFor(ext) + + uiUser, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-ui-user") + Expect(ok).To(BeTrue()) + Expect(uiSettingsRules(uiUser)).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + })) + Expect(lmaResourceNames(uiUser)).NotTo(ContainElement("runtime")) + + networkAdmin, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-network-admin") + Expect(ok).To(BeTrue()) + Expect(uiSettingsRules(networkAdmin)).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get", "patch", "update"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + })) + Expect(lmaResourceNames(networkAdmin)).NotTo(ContainElement("runtime")) + }) + }) + + It("queues the enterprise RBAC for deletion when not a management cluster", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + _, del := renderAPIServer(ci, ri, apiServerKeyPair(ci)) + _, ok := extensions.FindObject[*rbacv1.ClusterRole](del, render.ManagedClustersWatchClusterRoleName) + Expect(ok).To(BeTrue()) + }) + + Context("management cluster", func() { + It("adds the tunnel args and the managed-cluster-watch and secrets RBAC", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil, + &operatorv1.ManagementCluster{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ManagementClusterSpec{ + Address: "example.com:1234", + TLS: &operatorv1.TLS{SecretName: render.VoltronTunnelSecretName}, + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: render.VoltronTunnelSecretName, Namespace: common.OperatorNamespace()}, + Data: map[string][]byte{"cert": []byte("a"), "key": []byte("b")}, + }, + ) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(ci, ri, apiServerKeyPair(ci)) + + dp := apiServerDeployment(objs) + apiCtr := container(dp, render.APIServerContainerName) + Expect(apiCtr).NotTo(BeNil()) + Expect(apiCtr.Args).To(ContainElement("--enable-managed-clusters-create-api=true")) + Expect(apiCtr.Args).To(ContainElement("--managementClusterAddr=example.com:1234")) + Expect(apiCtr.Args).To(ContainElement("--tunnelSecretName=" + render.VoltronTunnelSecretName)) + + _, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.ManagedClustersWatchClusterRoleName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.Role](objs, render.APIServerSecretsRBACName) + Expect(ok).To(BeTrue()) + }) + }) + + Context("managed cluster", func() { + It("adds the external Linseed rolebinding and the query server token volume", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil, + &operatorv1.ManagementClusterConnection{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + }, + ) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(ci, ri, apiServerKeyPair(ci)) + + _, ok := extensions.FindObject[*rbacv1.RoleBinding](objs, "tigera-linseed") + Expect(ok).To(BeTrue()) + + dp := apiServerDeployment(objs) + Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", render.LinseedTokenVolumeName))) + qs := container(dp, render.TigeraAPIServerQueryServerContainerName) + Expect(qs).NotTo(BeNil()) + Expect(qs.Env).To(ContainElement(HaveField("Name", "LINSEED_TOKEN"))) + }) + }) + + Context("multi-tenant management cluster", func() { + // renderMultiTenantAPIServer mirrors renderAPIServer but sets MultiTenant on the render + // config, so the modifier takes the multi-tenant RBAC branch. + renderMultiTenantAPIServer := func(ci controller.Inputs, ri render.Inputs, kp certificatemanagement.KeyPairInterface) ([]client.Object, []client.Object) { + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: true, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: ci.RenderInputs.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: kp, + TrustedBundle: ri.TrustedBundle, + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + MultiTenant: true, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + + return ext.APIServer().Modify(extensionstest.APIServerStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: cfg}, ri).Objects() + } + + tenant := func(namespace string) *operatorv1.Tenant { + return &operatorv1.Tenant{ + ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: namespace}, + Spec: operatorv1.TenantSpec{ID: namespace}, + } + } + + multiTenantExt := func() extensions.Extensions { + return enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{MultiTenant: true}) + } + + It("does not render tigera-network-admin, so the RBAC management gate needs no tenancy term", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil, rbacManagementGate("true")) + eci, _, err := multiTenantExt().APIServer().ExtendInputs(ctx, ci) + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderMultiTenantAPIServer(ci, eci.RenderInputs, apiServerKeyPair(ci)) + _, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-network-admin") + Expect(ok).To(BeFalse()) + }) + + It("grants each tenant's calico-apiserver service account least-privilege Linseed access", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil, tenant("tenant-a"), tenant("tenant-b")) + eci, _, err := multiTenantExt().APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderMultiTenantAPIServer(ci, ri, apiServerKeyPair(ci)) + + // A dedicated, Linseed-only ClusterRole. + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-apiserver-linseed-access") + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ConsistOf(rbacv1.PolicyRule{ + APIGroups: []string{"linseed.tigera.io"}, + Resources: []string{"policyactivity"}, + Verbs: []string{"get"}, + })) + + // A single ClusterRoleBinding with one calico-apiserver ServiceAccount subject per tenant + // namespace. Linseed authorizes with a cluster-scoped SubjectAccessReview, so this must be a + // ClusterRoleBinding. + crb, ok := extensions.FindObject[*rbacv1.ClusterRoleBinding](objs, "calico-apiserver-linseed-access") + Expect(ok).To(BeTrue()) + Expect(crb.RoleRef.Name).To(Equal("calico-apiserver-linseed-access")) + Expect(crb.Subjects).To(ConsistOf( + rbacv1.Subject{Kind: "ServiceAccount", Name: render.APIServerServiceAccountName, Namespace: "tenant-a"}, + rbacv1.Subject{Kind: "ServiceAccount", Name: render.APIServerServiceAccountName, Namespace: "tenant-b"}, + )) + + // The zero-tenant user/network-admin roles are not installed in multi-tenant mode. + _, ok = extensions.FindObject[*rbacv1.ClusterRole](objs, "tigera-ui-user") + Expect(ok).To(BeFalse()) + }) + + It("queues the Linseed-access RBAC for deletion in zero-tenant mode", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + _, del := renderAPIServer(ci, ri, apiServerKeyPair(ci)) + _, ok := extensions.FindObject[*rbacv1.ClusterRole](del, "calico-apiserver-linseed-access") + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.ClusterRoleBinding](del, "calico-apiserver-linseed-access") + Expect(ok).To(BeTrue()) + }) + }) + + Context("v3-CRD mode (no aggregation server)", func() { + It("renders the deployment skeleton with the query server and pulls it out of the delete list", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: false, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: ci.RenderInputs.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: apiServerKeyPair(ci), + TrustedBundle: ri.TrustedBundle, + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + + // The base queues the deployment objects for deletion in v3-CRD mode. + _, ok := extensions.FindObject[*appsv1.Deployment](del, render.APIServerName) + Expect(ok).To(BeTrue()) + + create, del = ext.APIServer().Modify(extensionstest.APIServerStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: cfg}, ri).Objects() + + // After the modifier, the deployment (with the query server container) is in the + // create list and out of the delete list. + dp, ok := extensions.FindObject[*appsv1.Deployment](create, render.APIServerName) + Expect(ok).To(BeTrue()) + Expect(container(dp, render.TigeraAPIServerQueryServerContainerName)).NotTo(BeNil()) + _, ok = extensions.FindObject[*appsv1.Deployment](del, render.APIServerName) + Expect(ok).To(BeFalse()) + }) + + It("registers no APIService", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: false, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: ci.RenderInputs.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: apiServerKeyPair(ci), + TrustedBundle: ri.TrustedBundle, + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + + create, _ = ext.APIServer().Modify(extensionstest.APIServerStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: cfg}, ri).Objects() + + for _, r := range create { + Expect(r.GetObjectKind().GroupVersionKind().Kind).NotTo(Equal("APIService"), + "unexpected APIService registered in v3 CRD mode: %s", r.GetName()) + } + }) + }) + + Context("sidecar / L7 injection", func() { + applicationLayerSidecar := func() *operatorv1.ApplicationLayer { + enabled := operatorv1.SidecarEnabled + return &operatorv1.ApplicationLayer{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ApplicationLayerSpec{SidecarInjection: &enabled}, + } + } + + It("adds the L7 admission controller container, the sidecar webhook, and the L7 service port", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil, applicationLayerSidecar()) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + objs, _ := renderAPIServer(ci, ri, apiServerKeyPair(ci)) + + dp := apiServerDeployment(objs) + Expect(container(dp, render.L7AdmissionControllerContainerName)).NotTo(BeNil()) + + _, ok := extensions.FindObject[*admregv1.MutatingWebhookConfiguration](objs, common.SidecarMutatingWebhookConfigName) + Expect(ok).To(BeTrue()) + + svc, ok := extensions.FindObject[*corev1.Service](objs, render.APIServerServiceName) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(ContainElement(HaveField("Name", render.L7AdmissionControllerPortName))) + }) + + It("pulls the sidecar webhook out of the delete list", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil, applicationLayerSidecar()) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + _, del := renderAPIServer(ci, ri, apiServerKeyPair(ci)) + _, ok := extensions.FindObject[*admregv1.MutatingWebhookConfiguration](del, common.SidecarMutatingWebhookConfigName) + Expect(ok).To(BeFalse()) + }) + }) +}) + +var _ = Describe("API server enterprise policy modifier", func() { + applyPolicy := func(ci controller.Inputs, ri render.Inputs) *v3.NetworkPolicy { + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: true, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: ci.RenderInputs.Installation, + APIServer: &operatorv1.APIServerSpec{}, + } + comp := render.APIServerPolicy(cfg) + create, del := comp.Objects() + objs, _ := ext.APIServer().Modify(extensionstest.APIServerPolicyStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: cfg}, ri).Objects() + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.APIServerPolicyName) + Expect(ok).To(BeTrue()) + return policy + } + + It("leaves the egress rules as the base when no OIDC key validator is configured", func() { + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + policy := applyPolicy(ci, ri) + // The trailing rule remains the Pass rule (no OIDC egress rule inserted). + n := len(policy.Spec.Egress) + Expect(n).To(BeNumerically(">", 0)) + Expect(policy.Spec.Egress[n-1].Action).To(Equal(v3.Pass)) + }) + + It("adds the L7 admission controller ingress port when sidecar injection is enabled", func() { + enabled := operatorv1.SidecarEnabled + ci := apiServerControllerInputs(operatorv1.CalicoEnterprise, nil, &operatorv1.ApplicationLayer{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.ApplicationLayerSpec{SidecarInjection: &enabled}, + }) + eci, _, err := ext.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + policy := applyPolicy(ci, ri) + // Every ingress rule should carry the L7 admission controller port. + found := false + for _, rule := range policy.Spec.Ingress { + for _, p := range rule.Destination.Ports { + if p.MinPort == uint16(render.L7AdmissionControllerPort) { + found = true + } + } + } + Expect(found).To(BeTrue(), "expected the L7 admission controller ingress port") + }) +}) + +// cleanupAPIServer behaviour for the Calico variant: the base render component carries +// the enterprise cleanup modifier, which queues the enterprise RBAC for deletion. +var _ = Describe("API server Calico-variant cleanup", func() { + It("queues the enterprise RBAC for deletion", func() { + ci := apiServerControllerInputs(operatorv1.Calico, nil) + eci, _, err := calicoExt.APIServer().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + cfg := &render.APIServerConfiguration{ + RequiresAggregationServer: true, + K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, + Installation: ci.RenderInputs.Installation, + APIServer: &operatorv1.APIServerSpec{}, + TLSKeyPair: apiServerKeyPair(ci), + KubernetesVersion: &common.VersionInfo{Major: 1, Minor: 31}, + } + comp, err := render.APIServer(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + _, del = calicoExt.APIServer().Modify(extensionstest.APIServerStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: cfg}, ri).Objects() + + _, ok := extensions.FindObject[*rbacv1.ClusterRole](del, "tigera-ui-user") + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.ClusterRole](del, "calico-apiserver") + Expect(ok).To(BeTrue()) + }) +}) diff --git a/pkg/enterprise/apiserver/suite_test.go b/pkg/enterprise/apiserver/suite_test.go new file mode 100644 index 0000000000..beb62b9b38 --- /dev/null +++ b/pkg/enterprise/apiserver/suite_test.go @@ -0,0 +1,41 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apiserver_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" +) + +// calicoExt is the same Enterprise build running as Calico. +var ( + ext = enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{}) + calicoExt = enterprise.New(operatorv1.Calico, eoptions.Options{}) +) + +// ctx is the reconcile context the specs pass to the extension hooks. +var ctx = context.Background() + +func TestAPIServer(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/apiserver Suite") +} diff --git a/pkg/enterprise/clusterconnection/extension.go b/pkg/enterprise/clusterconnection/extension.go new file mode 100644 index 0000000000..fc4204bdfa --- /dev/null +++ b/pkg/enterprise/clusterconnection/extension.go @@ -0,0 +1,143 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection + +import ( + "context" + "fmt" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/controller/utils/imageset" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var log = logf.Log.WithName("clusterconnection-controller") + +// Extension is the Calico Enterprise behavior for the clusterconnection controller +// and the guardian components it renders. +type Extension struct { + variant operatorv1.ProductVariant +} + +var _ extensions.ClusterConnectionExtension = &Extension{} + +// New returns the clusterconnection extension for the variant the operator resolved. +func New(variant operatorv1.ProductVariant) *Extension { + return &Extension{variant: variant} +} + +// Modify dispatches over the components the clusterconnection controller renders. +func (e *Extension) Modify(c render.Component, ri render.Inputs) render.Component { + switch t := c.(type) { + case render.GuardianComponent: + return extensions.Decorate(c, ri, e.variant, func(objs, del []client.Object) ([]client.Object, []client.Object) { + return modifyGuardian(ri, t.GuardianConfig(), objs, del) + }) + case render.GuardianPolicyComponent: + return extensions.Decorate(c, ri, e.variant, func(objs, del []client.Object) ([]client.Object, []client.Object) { + return modifyGuardianPolicy(ri, t.GuardianPolicyConfig(), objs, del) + }) + default: + return c + } +} + +// Watches registers the resources only Enterprise guardian renders from. +func (e *Extension) Watches(c ctrlruntime.Controller, cs kubernetes.Interface) error { + // The license gates whether this controller reconciles network policy. + go utils.WaitToAddLicenseKeyWatch(c, cs, log, nil) + + if err := c.WatchObject(&operatorv1.ManagementCluster{}, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + for _, secretName := range []string{ + render.PacketCaptureServerCert, + monitor.PrometheusServerTLSSecretName, + certificatemanagement.CASecretName, + } { + if err := utils.AddSecretsWatch(c, secretName, common.OperatorNamespace()); err != nil { + return err + } + } + return imageset.AddImageSetWatch(c) +} + +// ValidateAndDefault accepts the Enterprise-only fields and defaults impersonation +// to empty lists so Guardian renders a stable config. +func (e *Extension) ValidateAndDefault(cr *operatorv1.ManagementClusterConnection) error { + if cr.Spec.Impersonation == nil { + cr.Spec.Impersonation = &operatorv1.Impersonation{ + Users: []string{}, + Groups: []string{}, + ServiceAccounts: []string{}, + } + } + return nil +} + +func (e *Extension) validate(ctx context.Context, ci controller.Inputs) error { + managementCluster, err := utils.GetManagementCluster(ctx, ci.Client) + if err != nil { + return fmt.Errorf("error reading ManagementCluster: %w", err) + } + if managementCluster != nil { + return extensions.InvalidConfigf("having both a ManagementCluster and a ManagementClusterConnection is not supported") + } + return nil +} + +// ExtendInputs computes the Enterprise-specific Guardian inputs the controller +// reads back: the managed cluster version (CNXVersion) and whether the license +// permits the domain-based egress network policy. It creates no certificates, so it +// returns no managed keypairs. The OSS controller path supplies its own defaults +// when this hook is absent. +func (e *Extension) ExtendInputs(ctx context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) { + if err := e.validate(ctx, ci); err != nil { + return ci, nil, err + } + + clusterInformation, err := utils.FetchClusterInformation(ctx, ci.Client) + if err != nil { + return ci, nil, extensions.Degradedf(operatorv1.ResourceReadError, "error querying ClusterInformation: %s", err) + } + + // Ensure the license can support enterprise policy before enabling the + // domain-based egress rules. A missing license simply leaves them disabled. + var includeEgressNetworkPolicy bool + if license, err := utils.FetchLicenseKey(ctx, ci.Client); err == nil { + includeEgressNetworkPolicy = utils.IsFeatureActive(license, common.EgressAccessControlFeature) + } else if !k8serrors.IsNotFound(err) { + return ci, nil, extensions.Degradedf(operatorv1.ResourceReadError, "error querying license: %s", err) + } + + ci.RenderInputs.Extension = render.GuardianRenderData{ + Version: clusterInformation.Spec.CNXVersion, + IncludeEgressNetworkPolicy: includeEgressNetworkPolicy, + } + return ci, nil, nil +} diff --git a/pkg/enterprise/clusterconnection/extension_test.go b/pkg/enterprise/clusterconnection/extension_test.go new file mode 100644 index 0000000000..f6e6782f6f --- /dev/null +++ b/pkg/enterprise/clusterconnection/extension_test.go @@ -0,0 +1,139 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("clusterconnection enterprise controller extension", func() { + var cli client.Client + + // controllerInputs builds a Inputs selecting the enterprise + // clusterconnection hook against the given client. + controllerInputs := func() controller.Inputs { + return controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}, + }, + Client: cli, + } + } + + clusterInformation := func() *v3.ClusterInformation { + return &v3.ClusterInformation{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: v3.ClusterInformationSpec{CNXVersion: "v3.99.0", CalicoVersion: "v3.99.0-calico"}, + } + } + + newClient := func(objs ...client.Object) client.Client { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + return ctrlrfake.DefaultFakeClientBuilder(scheme).WithObjects(objs...).Build() + } + + Describe("configuration", func() { + It("rejects a cluster that is both a management and a managed cluster", func() { + cli = newClient(&operatorv1.ManagementCluster{ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}}) + _, _, err := ext.ClusterConnection().ExtendInputs(ctx, controllerInputs()) + reason, ok := extensions.DegradedReason(err) + Expect(ok).To(BeTrue()) + Expect(reason).To(Equal(operatorv1.ResourceValidationError)) + Expect(err.Error()).To(ContainSubstring("not supported")) + }) + + It("accepts impersonation and defaults it to empty lists", func() { + cr := &operatorv1.ManagementClusterConnection{} + Expect(ext.ClusterConnection().ValidateAndDefault(cr)).NotTo(HaveOccurred()) + Expect(cr.Spec.Impersonation).To(Equal(&operatorv1.Impersonation{ + Users: []string{}, + Groups: []string{}, + ServiceAccounts: []string{}, + })) + }) + + It("leaves impersonation the user set alone", func() { + cr := &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{Users: []string{"jane"}}, + }, + } + Expect(ext.ClusterConnection().ValidateAndDefault(cr)).NotTo(HaveOccurred()) + Expect(cr.Spec.Impersonation.Users).To(Equal([]string{"jane"})) + }) + + It("accepts a public CA, which enterprise guardian can trust", func() { + cr := &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + TLS: &operatorv1.ManagementClusterTLS{CA: operatorv1.CATypePublic}, + }, + } + Expect(ext.ClusterConnection().ValidateAndDefault(cr)).NotTo(HaveOccurred()) + }) + }) + + Describe("ExtendInputs", func() { + It("reports the managed cluster CNX version", func() { + cli = newClient(clusterInformation()) + eci, managed, err := ext.ClusterConnection().ExtendInputs(ctx, controllerInputs()) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + Expect(managed).To(BeEmpty()) + + data, ok := render.GuardianRenderDataFromInputs(ri) + Expect(ok).To(BeTrue()) + Expect(data.Version).To(Equal("v3.99.0")) + Expect(data.IncludeEgressNetworkPolicy).To(BeFalse()) + }) + + It("enables the egress network policy when the license has the feature", func() { + license := &v3.LicenseKey{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Status: v3.LicenseKeyStatus{Features: []string{common.EgressAccessControlFeature}}, + } + cli = newClient(clusterInformation(), license) + eci, _, err := ext.ClusterConnection().ExtendInputs(ctx, controllerInputs()) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + data, ok := render.GuardianRenderDataFromInputs(ri) + Expect(ok).To(BeTrue()) + Expect(data.IncludeEgressNetworkPolicy).To(BeTrue()) + }) + + It("errors when ClusterInformation is missing", func() { + cli = newClient() + _, _, err := ext.ClusterConnection().ExtendInputs(ctx, controllerInputs()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("ClusterInformation")) + }) + }) +}) diff --git a/pkg/enterprise/clusterconnection/guardian.go b/pkg/enterprise/clusterconnection/guardian.go new file mode 100644 index 0000000000..18e3a92957 --- /dev/null +++ b/pkg/enterprise/clusterconnection/guardian.go @@ -0,0 +1,658 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection + +import ( + "net" + "net/url" + + "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" + operatorurl "github.com/tigera/operator/pkg/url" +) + +// modifyGuardianPolicy replaces the core OSS guardian network policy with the +// enterprise management-cluster policy. Building the enterprise egress rules can +// fail (proxy URL parsing); on failure we drop the policy entirely, matching the +// core behavior of omitting it rather than installing a partial policy. +func modifyGuardianPolicy(ri render.Inputs, gpc *render.GuardianConfiguration, objs, del []client.Object) ([]client.Object, []client.Object) { + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, render.GuardianPolicyName) + if !ok { + return objs, del + } + + spec, err := enterpriseGuardianPolicySpec(gpc) + if err != nil { + logrus.WithError(err).Error("Failed to build guardian network policy, policy will be omitted") + return removeObject(objs, policy), del + } + policy.Spec = spec + return objs, del +} + +func removeObject(objs []client.Object, drop client.Object) []client.Object { + out := objs[:0] + for _, o := range objs { + if o != drop { + out = append(out, o) + } + } + return out +} + +// enterpriseGuardianPolicySpec builds the network policy spec for guardian in a +// managed cluster: egress to the management cluster components and the tunnel +// destination(s), and ingress from the management-cluster components that reach +// back over the tunnel. +func enterpriseGuardianPolicySpec(gpc *render.GuardianConfiguration) (v3.NetworkPolicySpec, error) { + egressRules := []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: render.PacketCaptureEntityRule, + }, + } + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, gpc.OpenShift) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.KubeAPIServerEntityRule, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.PrometheusEntityRule, + }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: render.TigeraAPIServerEntityRule, + }, + }...) + + // Create an egress rule for each unique destination the guardian pods connect + // to. With multiple pods whose proxy settings differ, there are multiple + // destinations that must be allowed. + allowedDestinations := map[string]bool{} + for _, podProxyConfig := range render.ProcessPodProxies(gpc.PodProxies) { + var proxyURL *url.URL + var err error + if podProxyConfig != nil && podProxyConfig.HTTPSProxy != "" { + // The scheme is HTTPS, as we establish an mTLS session with the target. + // We expect the URL to be of the form host:port. + targetURL := &url.URL{Scheme: "https", Host: gpc.URL} + proxyURL, err = podProxyConfig.ProxyFunc()(targetURL) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + } + + var tunnelDestinationHostPort string + if proxyURL != nil { + proxyHostPort, err := operatorurl.ParseHostPortFromHTTPProxyURL(proxyURL) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + tunnelDestinationHostPort = proxyHostPort + } else { + // gpc.URL has host:port form. + tunnelDestinationHostPort = gpc.URL + } + + if allowedDestinations[tunnelDestinationHostPort] { + continue + } + + host, port, err := net.SplitHostPort(tunnelDestinationHostPort) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + parsedPort, err := numorstring.PortFromString(port) + if err != nil { + return v3.NetworkPolicySpec{}, err + } + parsedIP := net.ParseIP(host) + if parsedIP == nil { + // Domain-based egress rules require the EgressAccessControl license feature. + if !gpc.IncludeEgressNetworkPolicy { + continue + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Domains: []string{host}, + Ports: []numorstring.Port{parsedPort}, + }, + }) + allowedDestinations[tunnelDestinationHostPort] = true + } else { + netSuffix := "/32" + if parsedIP.To4() == nil { + netSuffix = "/128" + } + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Nets: []string{parsedIP.String() + netSuffix}, + Ports: []numorstring.Port{parsedPort}, + }, + }) + allowedDestinations[tunnelDestinationHostPort] = true + } + } + + egressRules = append(egressRules, v3.Rule{Action: v3.Pass}) + + dest := v3.EntityRule{Ports: networkpolicy.Ports(render.GuardianTargetPort)} + ingressRules := []v3.Rule{ + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.FluentBitSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.IntrusionDetectionSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Source: render.IntrusionDetectionInstallerSourceEntityRule, Destination: dest}, + {Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, Destination: dest}, + } + + return v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(render.GuardianName), + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingressRules, + Egress: egressRules, + }, nil +} + +// modifyGuardian layers Calico Enterprise behavior onto the rendered guardian +// objects: the secrets Role/RoleBinding and default UI settings, the +// elasticsearch/kibana service ports, the management-cluster-request cluster +// role rules (which replace the OSS rules), and the CA bundle env vars. +func modifyGuardian(ri render.Inputs, cfg *render.GuardianConfiguration, objs, del []client.Object) ([]client.Object, []client.Object) { + gc := guardianInputsFrom(cfg) + + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.GuardianClusterRoleName); ok { + role.Rules = guardianEnterpriseRules(gc) + } + + if svc, ok := extensions.FindObject[*corev1.Service](objs, render.GuardianServiceName); ok { + svc.Spec.Ports = append(svc.Spec.Ports, guardianEnterpriseServicePorts()...) + } + + if dep, ok := extensions.FindObject[*appsv1.Deployment](objs, render.GuardianDeploymentName); ok { + addGuardianEnterpriseEnv(gc, dep) + } + + return append(objs, + guardianSecretsRole(), + guardianSecretRoleBinding(), + // Default UI settings for this managed cluster. + render.ManagerClusterWideSettingsGroup(), + render.ManagerUserSpecificSettingsGroup(), + render.ManagerClusterWideTigeraLayer(), + render.ManagerClusterWideDefaultView(), + ), del +} + +// guardianInputs is what the guardian modifier needs beyond the installation: the +// impersonation config, the platform, and the CA bundle path the env vars reference. +type guardianInputs struct { + OpenShift bool + Impersonation *operatorv1.Impersonation + TrustedBundleMountPath string +} + +func guardianInputsFrom(cfg *render.GuardianConfiguration) guardianInputs { + var impersonation *operatorv1.Impersonation + if cfg.ManagementClusterConnection != nil { + impersonation = cfg.ManagementClusterConnection.Spec.Impersonation + } + + in := guardianInputs{OpenShift: cfg.OpenShift, Impersonation: impersonation} + if cfg.TrustedCertBundle != nil { + in.TrustedBundleMountPath = cfg.TrustedCertBundle.MountPath() + } + + return in +} + +// guardianEnterpriseRules are the cluster role rules guardian needs in Calico +// Enterprise. They wholly replace the OSS rules: the management cluster drives +// guardian over the tunnel, so it needs the union of the rules its components +// require, plus any configured impersonation and the OpenShift SCC. +func guardianEnterpriseRules(gc guardianInputs) []rbacv1.PolicyRule { + var rules []rbacv1.PolicyRule + + if imp := gc.Impersonation; imp != nil { + if imp.Users != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"users"}, + ResourceNames: imp.Users, + Verbs: []string{"impersonate"}, + }) + } + if imp.Groups != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"groups"}, + ResourceNames: imp.Groups, + Verbs: []string{"impersonate"}, + }) + } + if imp.ServiceAccounts != nil { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + ResourceNames: imp.ServiceAccounts, + Verbs: []string{"impersonate"}, + }) + } + } + + rules = append(rules, rulesForManagementClusterRequests(gc.OpenShift)...) + + if gc.OpenShift { + rules = append(rules, rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.NonRootV2}, + }) + } + + return rules +} + +func guardianEnterpriseServicePorts() []corev1.ServicePort { + return []corev1.ServicePort{ + { + Name: "elasticsearch", + Port: 9200, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "kibana", + Port: 5601, + TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: 8080}, + Protocol: corev1.ProtocolTCP, + }, + } +} + +func addGuardianEnterpriseEnv(gc guardianInputs, dep *appsv1.Deployment) { + c := render.MustContainer(&dep.Spec.Template.Spec, render.GuardianContainerName) + c.Env = append(c.Env, + corev1.EnvVar{Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + corev1.EnvVar{Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: gc.TrustedBundleMountPath}, + ) +} + +// guardianSecretsRole creates a Role that allows the management cluster to +// provision secrets to the tigera-operator Namespace, used to push secrets the +// managed cluster needs to access / authenticate with the management cluster. +func guardianSecretsRole() *rbacv1.Role { + return &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretsRole, + Namespace: common.OperatorNamespace(), + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"create", "delete", "deletecollection", "update"}, + }, + }, + } +} + +func guardianSecretRoleBinding() *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretsRoleBindingName, + Namespace: common.OperatorNamespace(), + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: render.GuardianSecretsRole, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: render.GuardianServiceAccountName, + Namespace: render.GuardianNamespace, + }, + }, + } +} + +// rulesForManagementClusterRequests returns the set of RBAC rules guardian needs +// to satisfy requests from the management cluster over the tunnel. +func rulesForManagementClusterRequests(isOpenShift bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + // Common rules required to handle requests from multiple components in the management cluster. + { + // ID uses read-only permissions and kube-controllers uses both read and write verbs. + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + // Allows Linseed to watch namespaces before copying its token. + // Also enables PolicyRecommendation to watch namespaces, + // and Manager/kube-controllers to list them. + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // kube-controllers watches Nodes to monitor for deletions. + // Manager performs a list operation on Nodes. + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // kube-controllers watches Pods to verify existence for IPAM garbage collection. + // Manager performs get operations on Pods. + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Manager uses list; kube-controllers uses 'get', 'list', 'watch', 'update'. + APIGroups: []string{""}, + Resources: []string{"services"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + // Needed by kube-controllers to validate licenses; also used by ID. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch"}, + }, + { + // Manager uses list; PolicyRecommendation & ID uses all verbs. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalnetworksets", + "networkpolicies", + "tier.networkpolicies", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + { + // Manager uses list; PolicyRecommendation uses all verbs. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + // Rules needed by guardian to handle manager authorization reviews. + { + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings", "roles", "rolebindings"}, + Verbs: []string{"list", "get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettings", "uisettingsgroups"}, + Verbs: []string{"list", "get"}, + }, + + // Rules needed by guardian to handle other manager requests. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"list"}, + }, + { + // Allow query server talk to Prometheus via the manager user. + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{ + "calico-node-prometheus:9090", + "https:calico-api:8080", + }, + Verbs: []string{"create", "get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"daemonsets", "replicasets", "statefulsets"}, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"authentication.k8s.io"}, + Resources: []string{"tokenreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"authorization.k8s.io"}, + Resources: []string{"subjectaccessreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"policy.networking.k8s.io"}, + Resources: []string{ + "clusternetworkpolicies", + "adminnetworkpolicies", + "baselineadminnetworkpolicies", + }, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"alertexceptions"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + ResourceNames: []string{"default"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalnetworkpolicies", + "networksets", + "stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "tier.globalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + }, + Verbs: []string{"list"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"list"}, + }, + + // Rules needed by guardian to handle policy recommendation requests. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "policyrecommendationscopes", + "policyrecommendationscopes/status", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + + // Rules needed by guardian to handle calico-kube-controller requests. + { + // Nodes are watched to monitor for deletions. + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"services/status"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + // Needs to manage hostendpoints. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"hostendpoints"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + { + // Needs access to update clusterinformations. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + // Needs to manipulate kubecontrollersconfiguration, which contains its config. + // It creates a default if none exists, and updates status as well. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"kubecontrollersconfigurations"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"tiers"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"crd.projectcalico.org", "projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"remoteclusterconfigurations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"create", "get", "list", "update", "watch"}, + }, + { + // Grant permissions to access ClusterInformation resources in managed clusters. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"clusterinformations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"usage.tigera.io"}, + Resources: []string{"licenseusagereports"}, + Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, + }, + + // Rules needed by guardian to handle Intrusion detection requests. + { + APIGroups: []string{""}, + Resources: []string{"podtemplates"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments"}, + Verbs: []string{"get"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"alertexceptions"}, + Verbs: []string{"get", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"securityeventwebhooks"}, + Verbs: []string{"get", "list", "update", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "globalalerts", + "globalalerts/status", + "globalthreatfeeds", + "globalthreatfeeds/status", + }, + Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, + }, + // Rules needed to fetch the compliance reports + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"globalreporttypes", "globalreports"}, + Verbs: []string{"get", "list", "watch"}, + }, + } + + // Rules needed by policy recommendation in openshift. + if isOpenShift { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{securitycontextconstraints.HostNetworkV2}, + }, + ) + } + + return rules +} diff --git a/pkg/enterprise/clusterconnection/guardian_render_test.go b/pkg/enterprise/clusterconnection/guardian_render_test.go new file mode 100644 index 0000000000..66e06514ed --- /dev/null +++ b/pkg/enterprise/clusterconnection/guardian_render_test.go @@ -0,0 +1,442 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/pkg/render/testutils" +) + +// The guardian policy fixtures live next to the render package's testutils, so +// reference them relative to this enterprise subpackage. +const ( + clusterDomain = "cluster.local" + guardianPolicyJSON = "../../render/testutils/expected_policies/guardian.json" + guardianPolicyOCPJSON = "../../render/testutils/expected_policies/guardian_ocp.json" +) + +// guardianObjects renders the guardian component and runs the enterprise extension +// over it, the way the clusterconnection controller does. +func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { + g := render.Guardian(cfg) + ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) + objs, _ := g.Objects() + ri := render.Inputs{Installation: cfg.Installation} + out, _ := ext.ClusterConnection().Modify(extensionstest.GuardianStub{StubComponent: extensionstest.StubComponent{Create: objs, Delete: nil}, Cfg: cfg}, ri).Objects() + return out +} + +var _ = Describe("Guardian enterprise rendering tests", func() { + var cfg *render.GuardianConfiguration + var g render.Component + var resources []client.Object + var deleteResources []client.Object + + createGuardianConfig := func(i operatorv1.InstallationSpec, addr string, openshift bool) *render.GuardianConfiguration { + i.Variant = operatorv1.CalicoEnterprise + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretName, + Namespace: common.OperatorNamespace(), + }, + Data: map[string][]byte{ + "cert": []byte("foo"), + "key": []byte("bar"), + }, + } + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + bundle := certificateManager.CreateTrustedBundle() + + return &render.GuardianConfiguration{ + URL: addr, + PullSecrets: []*corev1.Secret{{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "pull-secret", + Namespace: common.OperatorNamespace(), + }, + }}, + Installation: &i, + TunnelSecret: secret, + TrustedCertBundle: bundle, + OpenShift: openshift, + ManagementClusterConnection: &operatorv1.ManagementClusterConnection{}, + IncludeEgressNetworkPolicy: true, + } + } + + Context("Guardian component", func() { + renderGuardian := func(i operatorv1.InstallationSpec) { + cfg = createGuardianConfig(i, "127.0.0.1:1234", false) + g = render.Guardian(cfg) + Expect(g.ResolveImages(nil)).To(BeNil()) + resources, deleteResources = g.Objects() + // Run the extension the way the clusterconnection controller does, so these + // tests exercise the integrated output. + ri := render.Inputs{Installation: cfg.Installation} + resources, _ = ext.ClusterConnection().Modify(extensionstest.GuardianStub{StubComponent: extensionstest.StubComponent{Create: resources, Delete: nil}, Cfg: cfg}, ri).Objects() + } + + BeforeEach(func() { + renderGuardian(operatorv1.InstallationSpec{Registry: "my-reg/"}) + }) + + It("should layer the enterprise CA bundle env onto the guardian container", func() { + deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + c, ok := render.Container(&deployment.Spec.Template.Spec, render.GuardianContainerName) + Expect(ok).To(BeTrue()) + Expect(c.Env).To(ContainElement(HaveField("Name", "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH"))) + Expect(c.Env).To(ContainElement(HaveField("Name", "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH"))) + Expect(c.Env).To(ContainElement(HaveField("Name", "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH"))) + }) + + It("should render all resources for a managed cluster", func() { + expectedResources := []client.Object{ + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceAccountName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleBindingName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRole, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRoleBindingName, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: ""}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}}, + &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, + &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerUserSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, + &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsLayerTigera}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, + &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsViewDefault}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, + } + + expectedDeleteResources := []client.Object{ + &corev1.Namespace{TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, + &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, + &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager", Namespace: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-role"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-binding"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, + &netv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "guardian", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "networking.k8s.io/v1"}}, + } + + rtest.ExpectResources(resources, expectedResources) + rtest.ExpectResources(deleteResources, expectedDeleteResources) + + deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("my-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) + + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeFalse()) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(10001)) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsNonRoot).To(BeTrue()) + Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser).To(BeEquivalentTo(10001)) + Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.SeccompProfile).To(Equal( + &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + })) + Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities).To(Equal( + &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + )) + }) + + It("should render controlPlaneTolerations", func() { + t := corev1.Toleration{ + Key: "foo", + Operator: corev1.TolerationOpEqual, + Value: "bar", + } + renderGuardian(operatorv1.InstallationSpec{ + ControlPlaneTolerations: []corev1.Toleration{t}, + }) + deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElements(append(rmeta.TolerateCriticalAddonsAndControlPlane, t))) + }) + + It("should render toleration on GKE", func() { + renderGuardian(operatorv1.InstallationSpec{ + KubernetesProvider: operatorv1.ProviderGKE, + }) + deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(deployment).NotTo(BeNil()) + Expect(deployment.Spec.Template.Spec.Tolerations).To(ContainElements(corev1.Toleration{ + Key: "kubernetes.io/arch", + Operator: corev1.TolerationOpEqual, + Value: "arm64", + Effect: corev1.TaintEffectNoSchedule, + })) + }) + + It("should render guardian with unlimited impersonation", func() { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{ + Users: []string{}, + Groups: []string{}, + ServiceAccounts: []string{}, + }, + }, + } + + resources := guardianObjects(cfg) + Expect(resources).ToNot(BeNil()) + + clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(ok).To(BeTrue()) + + foundUserImp, foundGroupImp, foundSaImp := false, false, false + for _, rule := range clusterRole.Rules { + if rule.Verbs[0] == "impersonate" { + if rule.Resources[0] == "users" { + Expect(rule.ResourceNames).To(Equal([]string{})) + foundUserImp = true + } + if rule.Resources[0] == "groups" { + Expect(rule.ResourceNames).To(Equal([]string{})) + foundGroupImp = true + } + if rule.Resources[0] == "serviceaccounts" { + Expect(rule.ResourceNames).To(Equal([]string{})) + foundSaImp = true + } + } + } + + Expect(foundUserImp).To(BeTrue()) + Expect(foundGroupImp).To(BeTrue()) + Expect(foundSaImp).To(BeTrue()) + }) + + It("should render guardian with specific impersonation", func() { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{ + Users: []string{"foo"}, + Groups: []string{"bar"}, + ServiceAccounts: []string{"zaz"}, + }, + }, + } + + resources := guardianObjects(cfg) + Expect(resources).ToNot(BeNil()) + + clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(ok).To(BeTrue()) + + foundUserImp, foundGroupImp, foundSaImp := false, false, false + for _, rule := range clusterRole.Rules { + if rule.Verbs[0] == "impersonate" { + if rule.Resources[0] == "users" { + Expect(rule.ResourceNames).To(Equal([]string{"foo"})) + foundUserImp = true + } + if rule.Resources[0] == "groups" { + Expect(rule.ResourceNames).To(Equal([]string{"bar"})) + foundGroupImp = true + } + if rule.Resources[0] == "serviceaccounts" { + Expect(rule.ResourceNames).To(Equal([]string{"zaz"})) + foundSaImp = true + } + } + } + + Expect(foundUserImp).To(BeTrue()) + Expect(foundGroupImp).To(BeTrue()) + Expect(foundSaImp).To(BeTrue()) + }) + + It("should render guardian with specific no sa permissions but with user and group", func() { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{ + Users: []string{}, + Groups: []string{}, + }, + }, + } + + resources := guardianObjects(cfg) + Expect(resources).ToNot(BeNil()) + + clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(ok).To(BeTrue()) + + foundUserImp, foundGroupImp, foundSaImp := false, false, false + for _, rule := range clusterRole.Rules { + if rule.Verbs[0] == "impersonate" { + if rule.Resources[0] == "users" { + foundUserImp = true + } + if rule.Resources[0] == "groups" { + foundGroupImp = true + } + if rule.Resources[0] == "serviceaccounts" { + foundSaImp = true + } + } + } + + Expect(foundUserImp).To(BeTrue()) + Expect(foundGroupImp).To(BeTrue()) + Expect(foundSaImp).To(BeFalse()) + }) + }) + + It("should render SecurityContextConstrains properly when provider is OpenShift", func() { + cfg = createGuardianConfig(operatorv1.InstallationSpec{Registry: "my-reg/"}, "127.0.0.1:1234", false) + cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift + cfg.OpenShift = true + resources := guardianObjects(cfg) + + role := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + ResourceNames: []string{"nonroot-v2"}, + })) + }) + + Context("GuardianPolicy component", func() { + guardianPolicy := testutils.GetExpectedPolicyFromFile(guardianPolicyJSON) + guardianPolicyForOCP := testutils.GetExpectedPolicyFromFile(guardianPolicyOCPJSON) + + renderGuardianPolicy := func(addr string, openshift bool, variant operatorv1.ProductVariant, includeEgressNetworkPolicy bool) { + installation := operatorv1.InstallationSpec{ + Registry: "my-reg/", + } + cfg := createGuardianConfig(installation, addr, openshift) + cfg.Installation.Variant = variant + cfg.IncludeEgressNetworkPolicy = includeEgressNetworkPolicy + g, err := render.GuardianPolicy(cfg) + Expect(err).NotTo(HaveOccurred()) + objs, _ := g.Objects() + // Run the extension the way the clusterconnection controller does, so the + // enterprise policy is exercised. + ri := render.Inputs{Installation: cfg.Installation} + resources, _ = ext.ClusterConnection().Modify(extensionstest.GuardianPolicyStub{StubComponent: extensionstest.StubComponent{Create: objs, Delete: nil}, Cfg: cfg}, ri).Objects() + } + + Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { + It("should render Enterprise network policy without domain-based egress when IncludeEgressNetworkPolicy is false", func() { + // Enterprise variant with IncludeEgressNetworkPolicy=false should render a policy but skip domain-based egress rules + renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, false) + + policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + Expect(policy).NotTo(BeNil(), "Enterprise variant should always render a network policy when tier exists") + + // Verify it's the Enterprise policy (should have both Ingress and Egress types) + Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) + Expect(policy.Spec.Egress).NotTo(BeEmpty()) + + // Verify no domain-based egress rules are present + for _, rule := range policy.Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty(), + "Domain-based egress rules should not be present when IncludeEgressNetworkPolicy is false") + } + }) + + It("should render Enterprise network policy with domain-based egress when IncludeEgressNetworkPolicy is true", func() { + // Enterprise variant with IncludeEgressNetworkPolicy=true should render the full policy including domain-based egress + renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, true) + + policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + Expect(policy).NotTo(BeNil(), "Enterprise variant with IncludeEgressNetworkPolicy=true should render a network policy") + + // Verify it's the Enterprise policy (should have both Ingress and Egress types) + Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) + Expect(policy.Spec.Egress).NotTo(BeEmpty()) + + // Verify domain-based egress rule is present + hasDomainRule := false + for _, rule := range policy.Spec.Egress { + if len(rule.Destination.Domains) > 0 { + hasDomainRule = true + break + } + } + Expect(hasDomainRule).To(BeTrue(), "Domain-based egress rule should be present when IncludeEgressNetworkPolicy is true") + }) + }) + + Context("calico-system rendering", func() { + policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} + + getExpectedPolicy := func(name types.NamespacedName, scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { + if name.Name == "calico-system.guardian-access" && scenario.ManagedCluster { + return testutils.SelectPolicyByProvider(scenario, guardianPolicy, guardianPolicyForOCP) + } + + return nil + } + + DescribeTable("should render calico-system policy", + func(scenario testutils.CalicoSystemScenario) { + renderGuardianPolicy("127.0.0.1:1234", scenario.OpenShift, operatorv1.CalicoEnterprise, true) + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + expectedPolicy := getExpectedPolicy(policyName, scenario) + Expect(policy).To(Equal(expectedPolicy)) + }, + Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), + Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), + ) + + // The test matrix above validates against an IP-based management cluster address. + // Validate policy adaptation for domain-based management cluster address here. + It("should adapt Guardian policy if ManagementClusterAddr is domain-based", func() { + renderGuardianPolicy("mydomain.io:8080", false, operatorv1.CalicoEnterprise, true) + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + managementClusterEgressRule := policy.Spec.Egress[5] + Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{"mydomain.io"})) + Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(8080))) + }) + }) + }) +}) diff --git a/pkg/enterprise/clusterconnection/guardian_test.go b/pkg/enterprise/clusterconnection/guardian_test.go new file mode 100644 index 0000000000..6548b57398 --- /dev/null +++ b/pkg/enterprise/clusterconnection/guardian_test.go @@ -0,0 +1,114 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + client "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var _ = Describe("guardian enterprise modifier", func() { + // newObjs returns the subset of rendered guardian objects the modifier touches. + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, Rules: []rbacv1.PolicyRule{{Verbs: []string{"get"}}}}, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "https", Port: 443}}}, + }, + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: render.GuardianContainerName}}, + }}}, + }, + } + } + + entIn := render.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + + guardianCfg := func() *render.GuardianConfiguration { + return &render.GuardianConfiguration{Installation: entIn.Installation} + } + + It("appends the secrets RBAC and UI settings", func() { + out, _ := ext.ClusterConnection().Modify(extensionstest.GuardianStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: guardianCfg()}, entIn).Objects() + _, ok := extensions.FindObject[*rbacv1.Role](out, render.GuardianSecretsRole) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.RoleBinding](out, render.GuardianSecretsRoleBindingName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*v3.UISettingsGroup](out, render.ManagerClusterSettings) + Expect(ok).To(BeTrue()) + }) + + It("adds the elasticsearch and kibana service ports", func() { + out, _ := ext.ClusterConnection().Modify(extensionstest.GuardianStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: guardianCfg()}, entIn).Objects() + svc, _ := extensions.FindObject[*corev1.Service](out, render.GuardianServiceName) + names := []string{} + for _, p := range svc.Spec.Ports { + names = append(names, p.Name) + } + Expect(names).To(ContainElements("https", "elasticsearch", "kibana")) + }) + + It("replaces the cluster role rules and adds impersonation", func() { + gc := guardianCfg() + gc.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + Impersonation: &operatorv1.Impersonation{Users: []string{"foo"}, Groups: []string{"bar"}}, + }, + } + out, _ := ext.ClusterConnection().Modify(extensionstest.GuardianStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: gc}, entIn).Objects() + role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) + + // The single OSS placeholder rule is gone, replaced by the enterprise set. + Expect(role.Rules).NotTo(ContainElement(rbacv1.PolicyRule{Verbs: []string{"get"}})) + Expect(role.Rules).To(ContainElement(HaveField("ResourceNames", Equal([]string{"foo"})))) + Expect(role.Rules).To(ContainElement(HaveField("ResourceNames", Equal([]string{"bar"})))) + }) + + It("adds the CA bundle env to the guardian container", func() { + gc := guardianCfg() + gc.TrustedCertBundle = certificatemanagement.CreateTrustedBundle(nil) + out, _ := ext.ClusterConnection().Modify(extensionstest.GuardianStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: gc}, entIn).Objects() + dep, _ := extensions.FindObject[*appsv1.Deployment](out, render.GuardianDeploymentName) + Expect(dep.Spec.Template.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{ + Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", + Value: gc.TrustedCertBundle.MountPath(), + })) + }) + + It("does nothing when the operator runs as Calico", func() { + ctx := render.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out, _ := calicoExt.ClusterConnection().Modify(extensionstest.GuardianStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, ctx).Objects() + Expect(out).To(HaveLen(len(newObjs()))) + role, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.GuardianClusterRoleName) + Expect(role.Rules).To(Equal([]rbacv1.PolicyRule{{Verbs: []string{"get"}}})) + }) +}) diff --git a/pkg/enterprise/clusterconnection/suite_test.go b/pkg/enterprise/clusterconnection/suite_test.go new file mode 100644 index 0000000000..bbc3581b3b --- /dev/null +++ b/pkg/enterprise/clusterconnection/suite_test.go @@ -0,0 +1,40 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clusterconnection_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" +) + +var ( + ext = enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{}) + calicoExt = enterprise.New(operatorv1.Calico, eoptions.Options{}) +) + +// ctx is the reconcile context the specs pass to the extension hooks. +var ctx = context.Background() + +func TestClusterConnection(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/clusterconnection Suite") +} diff --git a/pkg/enterprise/installation/core.go b/pkg/enterprise/installation/core.go new file mode 100644 index 0000000000..5514692802 --- /dev/null +++ b/pkg/enterprise/installation/core.go @@ -0,0 +1,301 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation + +import ( + "context" + "fmt" + "strings" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + rbacv1 "k8s.io/api/rbac/v1" + apiextenv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller" + + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/imports/crds" + "github.com/tigera/operator/pkg/render" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + "github.com/tigera/operator/pkg/render/common/rbacmanagement" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// installationRenderData is the controller-produced data the installation +// extension hands to its modifiers through Inputs.Extension. The node +// modifier type-asserts it back out. +type installationRenderData struct { + nodePrometheusTLS certificatemanagement.KeyPairInterface + + // kubeControllerTLS is the calico-kube-controllers metrics serving keypair; the + // kube-controllers modifier mounts it onto the deployment. + kubeControllerTLS certificatemanagement.KeyPairInterface + + // collectProcessPath mirrors LogCollector.Spec.CollectProcessPath being + // enabled; the node modifier uses it to set HostPID and the felix env. + collectProcessPath bool + + // calico-kube-controllers enterprise additions the kube-controllers modifier + // applies: the enterprise cluster role rules, the enterprise enabled controllers, + // and the WAF v3 (Gateway API add-on) surface. + kubeControllerRules []rbacv1.PolicyRule + kubeControllerControllers []string + + // rbacManagementEnabled mirrors the rbac-ui-config gate; the kube-controllers + // modifier uses it to create the rbacsync controller's namespaced Role/RoleBinding. + rbacManagementEnabled bool + + // managedCluster reports whether this is a managed cluster, which decides whether + // kube-controllers reaches the manager through Guardian or directly. + managedCluster bool + + // managementCluster reports whether this is a management cluster, which is what + // gives kube-controllers the managed-cluster watch binding. + managementCluster bool + + waf wafRenderData +} + +// installationData pulls the installation extension's render data back out of the +// render inputs, returning the zero value when none is set. +func installationData(ri render.Inputs) installationRenderData { + return render.ExtractExtensionData[installationRenderData](ri) +} + +func collectProcessPathEnabled(lc *operatorv1.LogCollector) bool { + return lc != nil && + lc.Spec.CollectProcessPath != nil && + *lc.Spec.CollectProcessPath == operatorv1.CollectProcessPathEnable +} + +// Validate rejects installation config Calico Enterprise does not support. +// ProductVersion is the Calico Enterprise release the operator reports in status. +func (e *Extension) ProductVersion() string { + return components.EnterpriseRelease +} + +// DefaultFelixConfiguration sets the Enterprise-only FelixConfiguration defaults. +// Some platforms run a DNS service that isn't named "kube-dns", so dnsTrustedServers +// needs a provider-specific default for Enterprise DNS logging to work. Returns +// whether it changed fc. +func (e *Extension) DefaultFelixConfiguration(install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) { + dnsService := "" + switch install.KubernetesProvider { + case operatorv1.ProviderOpenShift: + dnsService = "k8s-service:openshift-dns/dns-default" + case operatorv1.ProviderRKE2: + dnsService = "k8s-service:kube-system/rke2-coredns-rke2-coredns" + } + if dnsService == "" { + return false, nil + } + + felixDefault := "k8s-service:kube-dns" + trustedServers := []string{dnsService} + // Keep any other values that are already configured, excepting the value we are + // setting and the kube-dns default. + existingSetting := "" + if fc.Spec.DNSTrustedServers != nil { + existingSetting = strings.Join(*fc.Spec.DNSTrustedServers, ",") + for _, server := range *fc.Spec.DNSTrustedServers { + if server != felixDefault && server != dnsService { + trustedServers = append(trustedServers, server) + } + } + } + if strings.Join(trustedServers, ",") == existingSetting { + return false, nil + } + fc.Spec.DNSTrustedServers = &trustedServers + return true, nil +} + +// Watches registers the enterprise resources the installation controller +// reconciles on. +func (e *Extension) Watches(c ctrlruntime.Controller) error { + for _, obj := range []client.Object{ + &operatorv1.ManagementCluster{}, + &operatorv1.ManagementClusterConnection{}, + &operatorv1.LogCollector{}, + // GatewayAPI.spec.extensions.waf.state gates the WAF v3 surface on calico-kube-controllers. + &operatorv1.GatewayAPI{}, + } { + if err := c.WatchObject(obj, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + } + // The switch gating the rbacsync controller and its RBAC. + if err := utils.AddConfigMapWatch(c, rbacmanagement.ConfigMapName, common.CalicoNamespace, &handler.EnqueueRequestForObject{}); err != nil { + return err + } + + // The core controller watches the Calico CRDs; these are the ones this variant adds. + if e.opts.ManageCRDs { + if err := utils.AddCRDWatches(c, enterpriseOnlyCRDs(e.opts.UseV3CRDs)); err != nil { + return err + } + } + + // es-kube-controllers includes the manager internal TLS secret in its bundle. + return utils.AddSecretsWatch(c, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) +} + +// ExtendInputs does the controller-side work the modifiers can't: creating and +// fetching the certificates that feed the trusted bundle. It returns the render +// inputs carrying the produced node prometheus keypair, and that keypair as one +// the controller should manage. +func (e *Extension) ExtendInputs(ctx context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) { + if err := ValidateReporterPort(ci.RenderInputs.FelixConfiguration); err != nil { + return ci, nil, err + } + + nodePrometheusTLS, err := ci.CertificateManager.GetOrCreateKeyPair( + ci.Client, + render.NodePrometheusTLSServerSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.CalicoNodeMetricsService, common.CalicoNamespace, ci.RenderInputs.ClusterDomain), + ) + if err != nil { + return ci, nil, fmt.Errorf("error creating node prometheus TLS certificate: %w", err) + } + if nodePrometheusTLS != nil { + ci.RenderInputs.TrustedBundle.AddCertificates(nodePrometheusTLS) + } + + // The calico-kube-controllers metrics endpoint is served with mTLS in + // Enterprise; the keypair is created here (cluster side effect) and mounted by + // the kube-controllers modifier. + kubeControllerTLS, err := ci.CertificateManager.GetOrCreateKeyPair( + ci.Client, + kubecontrollers.KubeControllerPrometheusTLSSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, ci.RenderInputs.ClusterDomain), + ) + if err != nil { + return ci, nil, fmt.Errorf("error creating kube-controllers metrics TLS certificate: %w", err) + } + if kubeControllerTLS != nil { + ci.RenderInputs.TrustedBundle.AddCertificates(kubeControllerTLS) + } + + logCollector, err := utils.GetLogCollector(ctx, ci.Client) + if err != nil { + return ci, nil, fmt.Errorf("error reading LogCollector: %w", err) + } + + // calico-kube-controllers enterprise additions: the WAF surface, the enterprise + // cluster role rules, and the enterprise enabled controllers. A managed cluster's + // kube-controllers needs an extra license-push rule. + managementClusterConnection, err := utils.GetManagementClusterConnection(ctx, ci.Client) + if err != nil { + return ci, nil, fmt.Errorf("error reading ManagementClusterConnection: %w", err) + } + + managementCluster, err := utils.GetManagementCluster(ctx, ci.Client) + if err != nil { + return ci, nil, fmt.Errorf("error reading ManagementCluster: %w", err) + } + if managementCluster != nil && managementClusterConnection != nil { + return ci, nil, extensions.InvalidConfigf("having both a ManagementCluster and a ManagementClusterConnection is not supported") + } + waf, wafWebhookTLS, err := buildWAFData(ctx, ci) + if err != nil { + return ci, nil, fmt.Errorf("error preparing WAF configuration: %w", err) + } + + // The rbacsync controller reconciles the ClusterRoles backing the Manager UI's + // RBAC management feature. + rbacManagementEnabled, err := utils.RBACManagementEnabled(ctx, ci.Client, e.variant, e.opts.MultiTenant) + if err != nil { + return ci, nil, fmt.Errorf("error reading the RBAC management UI ConfigMap: %w", err) + } + + ci.RenderInputs.Extension = installationRenderData{ + nodePrometheusTLS: nodePrometheusTLS, + kubeControllerTLS: kubeControllerTLS, + collectProcessPath: collectProcessPathEnabled(logCollector), + kubeControllerRules: calicoKubeControllersEnterpriseRules(waf.gatewayAPIPresent, managementClusterConnection != nil, rbacManagementEnabled), + kubeControllerControllers: calicoKubeControllersEnterpriseControllers(waf.gatewayAPIPresent, rbacManagementEnabled), + rbacManagementEnabled: rbacManagementEnabled, + managedCluster: managementClusterConnection != nil, + managementCluster: managementCluster != nil, + waf: waf, + } + + prometheusClientCert, err := ci.CertificateManager.GetCertificate(ci.Client, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace()) + if err != nil { + return ci, nil, fmt.Errorf("unable to fetch prometheus certificate: %w", err) + } + if prometheusClientCert != nil { + ci.RenderInputs.TrustedBundle.AddCertificates(prometheusClientCert) + } + + esgwCertificate, err := ci.CertificateManager.GetCertificate(ci.Client, relasticsearch.PublicCertSecret, common.OperatorNamespace()) + if err != nil { + return ci, nil, fmt.Errorf("failed to retrieve / validate %s: %w", relasticsearch.PublicCertSecret, err) + } + if esgwCertificate != nil { + ci.RenderInputs.TrustedBundle.AddCertificates(esgwCertificate) + } + + // es-kube-controllers talks to Voltron, so the shared bundle must trust the + // manager internal cert. + managerInternalTLS, err := ci.CertificateManager.GetCertificate(ci.Client, render.ManagerInternalTLSSecretName, common.OperatorNamespace()) + if err != nil { + return ci, nil, fmt.Errorf("failed to retrieve %s: %w", render.ManagerInternalTLSSecretName, err) + } + if managerInternalTLS != nil { + ci.RenderInputs.TrustedBundle.AddCertificates(managerInternalTLS) + } + + var managed []certificatemanagement.KeyPairInterface + if nodePrometheusTLS != nil { + managed = append(managed, nodePrometheusTLS) + } + if kubeControllerTLS != nil { + managed = append(managed, kubeControllerTLS) + } + if wafWebhookTLS != nil { + managed = append(managed, wafWebhookTLS) + } + return ci, managed, nil +} + +// enterpriseOnlyCRDs is the Calico Enterprise CRD set minus the Calico set the core +// controller already watches. +func enterpriseOnlyCRDs(useV3 bool) []*apiextenv1.CustomResourceDefinition { + calico := map[string]bool{} + for _, crd := range crds.GetCRDs(operatorv1.Calico, useV3) { + calico[crd.Name] = true + } + + var out []*apiextenv1.CustomResourceDefinition + for _, crd := range crds.GetCRDs(operatorv1.CalicoEnterprise, useV3) { + if !calico[crd.Name] { + out = append(out, crd) + } + } + return out +} diff --git a/pkg/enterprise/installation/core_test.go b/pkg/enterprise/installation/core_test.go new file mode 100644 index 0000000000..ac1a591c43 --- /dev/null +++ b/pkg/enterprise/installation/core_test.go @@ -0,0 +1,118 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "k8s.io/apimachinery/pkg/runtime" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/kubecontrollers" +) + +var _ = Describe("installation controller extension", func() { + It("rejects a zero prometheus reporter port", func() { + port := 0 + ci := newControllerInputs(operatorv1.CalicoEnterprise) + ci.RenderInputs.FelixConfiguration = &v3.FelixConfiguration{ + Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &port}, + } + _, _, err := ext.Installation().ExtendInputs(ctx, ci) + reason, ok := extensions.DegradedReason(err) + Expect(ok).To(BeTrue()) + Expect(reason).To(Equal(operatorv1.ResourceValidationError)) + }) + + DescribeTable("defaults dnsTrustedServers for providers whose DNS service isn't kube-dns", + func(provider operatorv1.Provider, expected []string) { + fc := &v3.FelixConfiguration{} + install := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider} + updated, err := ext.Installation().DefaultFelixConfiguration(install, fc) + Expect(err).NotTo(HaveOccurred()) + if expected == nil { + Expect(updated).To(BeFalse()) + Expect(fc.Spec.DNSTrustedServers).To(BeNil()) + return + } + Expect(updated).To(BeTrue()) + Expect(*fc.Spec.DNSTrustedServers).To(ConsistOf(expected)) + }, + Entry("OpenShift", operatorv1.ProviderOpenShift, []string{"k8s-service:openshift-dns/dns-default"}), + Entry("RKE2", operatorv1.ProviderRKE2, []string{"k8s-service:kube-system/rke2-coredns-rke2-coredns"}), + Entry("other providers keep the felix default", operatorv1.ProviderNone, nil), + ) + + It("does no felix defaulting when the operator runs as Calico", func() { + fc := &v3.FelixConfiguration{} + updated, err := calicoExt.Installation().DefaultFelixConfiguration(&operatorv1.InstallationSpec{Variant: operatorv1.Calico, KubernetesProvider: operatorv1.ProviderOpenShift}, fc) + Expect(err).NotTo(HaveOccurred()) + Expect(updated).To(BeFalse()) + Expect(fc.Spec.DNSTrustedServers).To(BeNil()) + }) + + It("manages the node prometheus and kube-controllers metrics keypairs for the enterprise variant", func() { + _, managed, err := ext.Installation().ExtendInputs(ctx, newControllerInputs(operatorv1.CalicoEnterprise)) + Expect(err).NotTo(HaveOccurred()) + names := []string{} + for _, kp := range managed { + names = append(names, kp.GetName()) + } + Expect(names).To(ConsistOf(render.NodePrometheusTLSServerSecret, kubecontrollers.KubeControllerPrometheusTLSSecret)) + }) + + It("is a no-op when the operator runs as Calico", func() { + _, managed, err := calicoExt.Installation().ExtendInputs(ctx, newControllerInputs(operatorv1.Calico)) + Expect(err).NotTo(HaveOccurred()) + Expect(managed).To(BeEmpty()) + }) +}) + +func newControllerInputs(variant operatorv1.ProductVariant, objs ...client.Object) controller.Inputs { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + for _, o := range objs { + Expect(c.Create(context.Background(), o)).NotTo(HaveOccurred()) + } + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + trustedBundle := certManager.CreateTrustedBundle() + + return controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: &operatorv1.InstallationSpec{Variant: variant}, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: trustedBundle, + ClusterDomain: "cluster.local", + }, + Client: c, + CertificateManager: certManager, + } +} diff --git a/pkg/enterprise/installation/extension.go b/pkg/enterprise/installation/extension.go new file mode 100644 index 0000000000..7098a33673 --- /dev/null +++ b/pkg/enterprise/installation/extension.go @@ -0,0 +1,83 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + eoptions "github.com/tigera/operator/pkg/enterprise/options" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/kubecontrollers" +) + +// Extension is the Calico Enterprise behavior for the installation controller and +// the components it renders. +type Extension struct { + variant operatorv1.ProductVariant + opts eoptions.Options + images *imageoverride.Overrides +} + +var _ extensions.InstallationExtension = &Extension{} + +// New returns the installation extension for the variant the operator resolved. +func New(variant operatorv1.ProductVariant, opts eoptions.Options) *Extension { + images := imageoverride.New() + images.Register(variant, render.ComponentNameNode, components.ComponentTigeraNode) + + // The node component renders the cni-plugins init container; its image resolves + // through its own override key. + images.Register(variant, render.ComponentNameCNIPlugins, components.ComponentTigeraCNIPlugins) + + if opts.Cloud { + // Calico Cloud runs kube-controllers from the combined image, which carries the + // Cloud behavior the mono image lacks. + images.Register(variant, render.ComponentNameKubeControllers, components.CalicoCloudImage()) + } + + return &Extension{variant: variant, opts: opts, images: images} +} + +func (e *Extension) Images() *imageoverride.Overrides { + return e.images +} + +// Modify dispatches over the components the installation controller renders. +func (e *Extension) Modify(c render.Component, ri render.Inputs) render.Component { + switch c.(type) { + case render.NodeComponent: + return extensions.Decorate(c, ri, e.variant, func(objs, del []client.Object) ([]client.Object, []client.Object) { + return modifyNode(ri, objs, del) + }) + case render.TyphaComponent: + return extensions.Decorate(c, ri, e.variant, func(objs, del []client.Object) ([]client.Object, []client.Object) { + return modifyTypha(ri, objs, del) + }) + case kubecontrollers.CalicoComponent: + return extensions.Decorate(c, ri, e.variant, func(objs, del []client.Object) ([]client.Object, []client.Object) { + return modifyKubeControllers(ri, objs, del) + }) + case kubecontrollers.CalicoPolicyComponent: + return extensions.Decorate(c, ri, e.variant, func(objs, del []client.Object) ([]client.Object, []client.Object) { + return modifyKubeControllersPolicy(ri, objs, del) + }) + default: + return c + } +} diff --git a/pkg/enterprise/installation/kubecontrollers.go b/pkg/enterprise/installation/kubecontrollers.go new file mode 100644 index 0000000000..6a86c58a75 --- /dev/null +++ b/pkg/enterprise/installation/kubecontrollers.go @@ -0,0 +1,705 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strconv" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/controller/gatewayapi" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/controller/utils/imageset" + "github.com/tigera/operator/pkg/dns" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/applicationlayer" + rcomp "github.com/tigera/operator/pkg/render/common/components" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/secret" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// modifyKubeControllersPolicy adds the WAF admission webhook ingress rule to the +// calico-kube-controllers calico-system network policy, so the kube-apiserver can +// reach the in-process webhook on :9443 (EV-6386). Without it the calico-system +// default-deny drops the apiserver->:9443 call and WAF admission times out. +func modifyKubeControllersPolicy(ri render.Inputs, objs, del []client.Object) ([]client.Object, []client.Object) { + data := installationData(ri) + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) + if !ok { + return objs, del + } + + // kube-controllers reaches the manager through Guardian on a managed cluster and + // directly otherwise. + manager := networkpolicy.DefaultHelper().ManagerEntityRule() + if data.managedCluster { + manager = render.GuardianEntityRule + } + policy.Spec.Egress = append(policy.Spec.Egress, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: manager, + }) + + if data.waf.enabled { + policy.Spec.Ingress = append(policy.Spec.Ingress, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), + }, + }) + } + return objs, del +} + +// modifyKubeControllers layers the full Calico Enterprise surface onto the rendered +// calico-kube-controllers objects: the enterprise cluster role rules, the enterprise +// enabled controllers, the metrics serving TLS, and the WAF v3 (Gateway API add-on) +// surface. The modifier only runs for the enterprise variant, so everything it adds +// is enterprise-only by construction - the base render carries none of it. The +// controller-side inputs (keypairs, the resolved wasm image, the pull secret) are +// produced by the installation hook and handed in through ri. +func modifyKubeControllers(ri render.Inputs, objs, del []client.Object) ([]client.Object, []client.Object) { + data := installationData(ri) + + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole); ok { + role.Rules = append(role.Rules, data.kubeControllerRules...) + } + + // Only a management cluster watches managed clusters. + if data.managementCluster { + objs = append(objs, rcomp.ClusterRoleBinding( + kubecontrollers.ManagedClustersWatchRoleBindingName, + render.ManagedClustersWatchClusterRoleName, + kubecontrollers.KubeControllerServiceAccount, + []string{common.CalicoNamespace}, + )) + } + + if dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController); ok { + modifyKubeControllersDeployment(ri, dp, data) + } + + // The WAF admission webhook surface (Service + ValidatingWebhookConfiguration), + // the wasm pull secret, and the wasm CA bundle. Created when WAF is enabled, + // deleted otherwise so toggling the extension off cleans them up. + webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(data.waf.caBundle) + if data.waf.enabled { + objs = append(objs, webhookObjs...) + if data.waf.pullSecret != nil { + objs = append(objs, secret.ToRuntimeObjects(secret.CopyToNamespace(common.CalicoNamespace, data.waf.pullSecret)...)...) + } + if data.waf.caCert != nil { + objs = append(objs, data.waf.caCert) + } + } else { + del = append(del, webhookObjs...) + } + + // The rbacsync controller's namespaced Role/RoleBinding. Deleted when RBAC + // management is off, so switching the gate off cleans them up. + if data.rbacManagementEnabled { + objs = append(objs, rbacSyncIDPGroupsRole()...) + } else { + del = append(del, rbacSyncIDPGroupsRole()...) + } + + return objs, del +} + +func modifyKubeControllersDeployment(ri render.Inputs, dp *appsv1.Deployment, data installationRenderData) { + spec := &dp.Spec.Template.Spec + if dp.Spec.Template.Annotations == nil { + dp.Spec.Template.Annotations = map[string]string{} + } + + if tls := data.kubeControllerTLS; tls != nil { + spec.Volumes = append(spec.Volumes, tls.Volume()) + dp.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() + } + if waf := data.waf; waf.enabled && waf.webhookTLS != nil { + spec.Volumes = append(spec.Volumes, waf.webhookTLS.Volume()) + } + + { + c := render.MustContainer(spec, kubecontrollers.KubeController) + + appendEnabledControllers(c, data.kubeControllerControllers) + c.Env = append(c.Env, enterpriseEnv(ri)...) + + if tls := data.kubeControllerTLS; tls != nil { + c.Env = append(c.Env, + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + ) + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + + // The applicationlayer WAF reconcilers are wired whenever the GatewayAPI CR is + // present (see calicoKubeControllersEnterpriseControllers), so they can tear + // down the EnvoyExtensionPolicies they generated when WAF is disabled. + // WAF_GATEWAY_EXTENSION_ENABLED tells them whether to program (enabled) or + // de-program (disabled) - EV-6751. Absent => the reconciler defaults to enabled, + // so an older operator that predates this var is unaffected. + if data.waf.gatewayAPIPresent { + c.Env = append(c.Env, corev1.EnvVar{Name: "WAF_GATEWAY_EXTENSION_ENABLED", Value: strconv.FormatBool(data.waf.enabled)}) + } + + if waf := data.waf; waf.enabled { + c.Env = append(c.Env, wafEnv(waf)...) + c.Ports = append(c.Ports, corev1.ContainerPort{ + Name: "waf-webhook", + ContainerPort: applicationlayer.WAFWebhookContainerPort, + Protocol: corev1.ProtocolTCP, + }) + if waf.webhookTLS != nil { + c.VolumeMounts = append(c.VolumeMounts, waf.webhookTLS.VolumeMount(rmeta.OSTypeLinux)) + if waf.webhookTLS.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, waf.webhookTLS.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + } + } + } +} + +// appendEnabledControllers folds the enterprise controllers into the existing +// ENABLED_CONTROLLERS env the base render set (node,loadbalancer). +func appendEnabledControllers(c *corev1.Container, extra []string) { + if len(extra) == 0 { + return + } + for i := range c.Env { + if c.Env[i].Name == "ENABLED_CONTROLLERS" { + c.Env[i].Value = c.Env[i].Value + "," + strings.Join(extra, ",") + return + } + } +} + +// enterpriseEnv is the static enterprise env for calico-kube-controllers. The +// modifier runs only for the enterprise variant, so these are never rendered for core. +func enterpriseEnv(ri render.Inputs) []corev1.EnvVar { + var env []corev1.EnvVar + if ri.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: ri.TrustedBundle.MountPath()}) + } + if in := ri.Installation; in != nil && in.CalicoNetwork != nil && in.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: in.CalicoNetwork.MultiInterfaceMode.Value()}) + } + return env +} + +// wafEnv is the WAF v3 env the kube-controllers binary consumes to program WAF policy +// attachments. WASM_IMAGE is the pre-resolved reference the hook produced. +func wafEnv(waf wafRenderData) []corev1.EnvVar { + var env []corev1.EnvVar + if waf.wasmImage != "" { + env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: waf.wasmImage}) + } + if waf.pullSecret != nil { + env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: waf.pullSecret.Name}) + } + if waf.caCert != nil { + env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: waf.caCert.Name}) + } + if waf.webhookTLS != nil { + env = append(env, corev1.EnvVar{Name: "WAF_WEBHOOK_CERT_DIR", Value: filepath.Dir(waf.webhookTLS.VolumeMountCertificateFilePath())}) + } + return env +} + +const ( + // WASMPullSecretName is the dedicated image-pull Secret (a merged copy of the + // install pull secrets) the WAF reconciler replicates into tenant namespaces for + // the Coraza wasm OCI pull. A dedicated name avoids clashing with the + // operator-managed tigera-pull-secret the GatewayAPI render also copies there (EV-6386). + WASMPullSecretName = "tigera-waf-pull-secret" + + // WASMCACertName is the dedicated CA-bundle ConfigMap the WAF reconciler + // replicates into tenant namespaces for the Coraza wasm OCI registry TLS check - + // a dedicated name avoids clashing with the operator-managed tigera-ca-bundle the + // GatewayAPI render also copies there (EV-6386). It is a renamed copy of the trusted bundle. + WASMCACertName = "tigera-waf-ca-bundle" +) + +// calicoKubeControllersEnterpriseRules are the enterprise cluster role rules layered +// onto calico-kube-controllers: the shared enterprise rules plus the calico-specific +// ones (federated endpoints, license usage reporting), and the rbacsync controller +// rules when RBAC management is enabled. +func calicoKubeControllersEnterpriseRules(gatewayAPIPresent, managedCluster, rbacManagementEnabled bool) []rbacv1.PolicyRule { + rules := entkubecontrollers.KubeControllersEnterpriseCommonRules(gatewayAPIPresent, managedCluster) + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"remoteclusterconfigurations"}, + Verbs: []string{"watch", "list", "get"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "update", "delete"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"usage.tigera.io"}, + Resources: []string{"licenseusagereports"}, + Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, + }, + ) + if rbacManagementEnabled { + rules = append(rules, rbacSyncControllerRules()...) + } + return rules +} + +// calicoKubeControllersEnterpriseControllers are the enterprise controllers added to +// the calico-kube-controllers ENABLED_CONTROLLERS list (on top of the base +// node,loadbalancer). applicationlayer is wired whenever the GatewayAPI CR is present, +// not only when WAF is enabled, so it stays running and can tear down the +// EnvoyExtensionPolicies it generated when WAF is disabled (it de-programs vs programs +// based on WAF_GATEWAY_EXTENSION_ENABLED - EV-6751). rbacsync is added only when RBAC +// management is enabled. +func calicoKubeControllersEnterpriseControllers(gatewayAPIPresent, rbacManagementEnabled bool) []string { + controllers := []string{"service", "federatedservices", "usage"} + if gatewayAPIPresent { + controllers = append(controllers, "applicationlayer") + } + if rbacManagementEnabled { + controllers = append(controllers, "rbacsync") + } + return controllers +} + +// rbacSyncIDPGroupsRole returns the Role + RoleBinding that grants rbacsync +// read access to the tigera-idp-groups ConfigMap in calico-system, its only +// namespaced dependency. +func rbacSyncIDPGroupsRole() []client.Object { + name := "calico-kube-controllers-rbac-sync" + return []client.Object{ + &rbacv1.Role{ + TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: common.CalicoNamespace}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + ResourceNames: []string{"tigera-idp-groups"}, + Verbs: []string{"get", "list", "watch"}, + }, + }, + }, + &rbacv1.RoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: common.CalicoNamespace}, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: kubecontrollers.KubeControllerServiceAccount, + Namespace: common.CalicoNamespace, + }, + }, + }, + } +} + +// rbacSyncControllerRules returns the cluster-scoped rules the rbacsync +// controller holds. The controller reconciles the ClusterRoles that back the +// Manager UI's RBAC management feature, and each rule below lets it manage the +// access one Calico Enterprise UI feature (and its view or modify state) +// requires. The controller runs only when RBAC management is enabled. +// +// Under Kubernetes' privilege-escalation guard the controller can only grant +// permissions it already holds, so each rule mirrors a grant made by one of the +// managed calico-ui-* ClusterRoles the rbacsync controller generates in +// calico-private (kube-controllers/pkg/controllers/rbacsync: resourceroles.go +// defines the calico-ui--{view,mod} and calico-ui-logs-view-* roles, +// tierroles.go the calico-ui-{np,gnp}-{view,mod}- and calico-ui-cluster- +// context roles). The comment on each rule names the managed role(s) it covers. +// +// Only the grants unique to the managed roles live here. Core resources those +// roles also grant (namespaces, nodes, services, pods, clusterinformations, +// hostendpoints, serviceaccounts, tiers) are already held by the common +// kube-controllers rules above, which satisfy the escalation guard for them. +func rbacSyncControllerRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + // RBAC management: the ClusterRoles and bindings the controller + // reconciles for the feature. Not a mirrored grant — this is the + // controller's own reconcile target for every managed calico-ui-* role. + { + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings", "rolebindings"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Network Policy tiers, view and modify: the per-tier and all-tiers + // Policies and Global Policies roles cover the tiers and tier-scoped + // (tier.*) policy resources. The plain networkpolicies and + // stagednetworkpolicies come from Policy Recommendations, which + // references them directly. Mirrors calico-ui-{np,gnp}-{view,mod}- + // (and -all), calico-ui-get-tier-* and calico-ui-policy-recommendations- + // {view,mod}. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "tiers", + "tier.networkpolicies", + "tier.stagednetworkpolicies", + "tier.globalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "networkpolicies", + "stagednetworkpolicies", + }, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Network Policy tiers, view and modify: Kubernetes network policies + // within a tier. Mirrors calico-ui-np-{view,mod}- (and -all). + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // The per-feature pages, view and modify: Dashboards, Managed Clusters, + // Global Network Sets, Network Sets, Policy Recommendations, Packet + // Captures, Alerts and Security Events, Threat Feeds, Compliance + // Reports, Webhooks, Deep Packet Inspection, and Egress Gateways. + // Mirrors the matching calico-ui--{view,mod} roles + // (e.g. calico-ui-managed-clusters-{view,mod}, calico-ui-alerts- + // {view,mod}, calico-ui-egress-gateways-{view,mod}). + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "uisettings", + "uisettingsgroups", + "globalnetworksets", + "networksets", + "managedclusters", + "policyrecommendationscopes", + "policyrecommendationscopes/status", + "deeppacketinspections", + "deeppacketinspections/status", + "egressgatewaypolicies", + "externalnetworks", + "globalalerts", + "globalalerts/status", + "globalalerttemplates", + "alertexceptions", + "globalthreatfeeds", + "globalthreatfeeds/status", + "globalreports", + "globalreports/status", + "globalreporttypes", + "packetcaptures", + "packetcaptures/files", + "securityeventwebhooks", + }, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Dashboards, view and modify: the cluster-settings and user-settings + // dashboard layouts stored on the UISettingsGroups data subresource. + // Mirrors calico-ui-dashboards-{view,mod}. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Manager UI load: the authorization self-check the UI runs on load. + // Packet Captures: authenticating a capture-file download. Mirrors the + // authorizationreviews grant on calico-ui-cluster-context and the + // authenticationreviews grant on calico-ui-packet-captures-{view,mod}. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"authorizationreviews", "authenticationreviews"}, + Verbs: []string{"create"}, + }, + // Manager UI load: Felix configuration read for cluster-wide settings. + // Mirrors calico-ui-cluster-context. + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"felixconfigurations"}, + Verbs: []string{"get", "list", "watch"}, + }, + // Webhooks, modify: creating and updating the Secret that stores the + // webhook credentials. Mirrors calico-ui-webhooks-mod. + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + ResourceNames: []string{"webhooks-secret"}, + Verbs: []string{"patch"}, + }, + // Logs, view: Flow, DNS, Audit, L7, and Events log access, per managed + // cluster and for the management cluster. Mirrors calico-ui-logs-view-* + // (all/audit/dns/events/flows/l7, plus their per-cluster and + // all-clusters variants). + { + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"cluster"}, + Verbs: []string{"get"}, + }, + // Manager UI load: the Compliance feature-enabled check. Mirrors the + // unscoped compliances grant on calico-ui-cluster-context. (The + // calico-ui-compliance-reports-{view,mod} roles also read compliances + // but scope it to the tigera-secure CR; this rule must stay unscoped to + // cover cluster-context, whose feature check is not resource-scoped.) + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"compliances"}, + Verbs: []string{"get"}, + }, + // Manager UI load: feature-enabled checks for Application Layer / WAF, + // Packet Capture, and Intrusion Detection. Mirrors calico-ui-cluster- + // context (which bundles the compliances check above into the same rule). + { + APIGroups: []string{"operator.tigera.io"}, + Resources: []string{"applicationlayers", "packetcaptureapis", "intrusiondetections"}, + Verbs: []string{"get"}, + }, + // Global Network Sets and Network Sets, view and modify: listing the + // pods a network set selects. Mirrors the pods grant on calico-ui- + // {global-network-sets,network-sets}-{view,mod} and calico-ui-service- + // graph-{view,mod}. (The common kube-controllers rules above already + // grant pods get/list/watch for IPAM GC, so this is also covered there.) + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"list"}, + }, + // Service Graph: the service accounts the flow view references. The only + // managed role granting serviceaccounts, so mirrors calico-ui-service- + // graph-{view,mod}. (That role also grants services, namespaces and + // hostendpoints, all covered by the common kube-controllers rules above.) + { + APIGroups: []string{""}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"get", "list"}, + }, + // Manager UI load: the statistics proxy to the Calico API server and + // the node Prometheus. Mirrors calico-ui-cluster-context. + { + APIGroups: []string{""}, + Resources: []string{"services/proxy"}, + ResourceNames: []string{"https:calico-api:8080", "calico-node-prometheus:9090"}, + Verbs: []string{"get", "create"}, + }, + } +} + +// wafRenderData is the controller-produced WAF v3 (Gateway API add-on) state the +// installation hook hands the kube-controllers modifier through the render inputs. +// The zero value (both flags false) means the modifier deletes the webhook objects +// and wires none of the WAF surface. +// +// The two flags are deliberately distinct (EV-6751). gatewayAPIPresent means the +// GatewayAPI CR exists regardless of waf.state; it keeps the applicationlayer +// controller wired, its EnvoyExtensionPolicy RBAC in place, and the +// WAF_GATEWAY_EXTENSION_ENABLED env present, so the controller can tear down the +// EnvoyExtensionPolicies it generated when WAF is turned off instead of losing its +// RBAC in the same reconcile. enabled means waf.state == Enabled and gates the +// active surface: the admission webhook, its serving cert, and the WASM env. +type wafRenderData struct { + gatewayAPIPresent bool + enabled bool + wasmImage string + pullSecret *corev1.Secret + caCert *corev1.ConfigMap + webhookTLS certificatemanagement.KeyPairInterface + caBundle []byte +} + +// buildWAFData reads the GatewayAPI CR and, when the WAF extension is enabled, +// produces everything the modifier needs that it can't compute itself: the resolved +// wasm image, the webhook serving keypair (also returned as a managed keypair), the +// merged wasm pull secret, the wasm CA bundle ConfigMap, and the operator CA PEM. +func buildWAFData(ctx context.Context, ci controller.Inputs) (wafRenderData, certificatemanagement.KeyPairInterface, error) { + gw, msg, err := gatewayapi.GetGatewayAPI(ctx, ci.Client) + if err != nil && !apierrors.IsNotFound(err) { + if msg != "" { + return wafRenderData{}, nil, fmt.Errorf("%s: %w", msg, err) + } + return wafRenderData{}, nil, err + } + if gw == nil { + return wafRenderData{}, nil, nil + } + // The GatewayAPI CR exists. Keep the WAF controller wired and its RBAC/env + // present even while WAF is disabled, so kube-controllers can tear down the + // EnvoyExtensionPolicies it generated (EV-6751). The active surface below is + // gated separately on waf.state == Enabled. + if !gw.Spec.IsWAFGatewayExtensionEnabled() { + return wafRenderData{gatewayAPIPresent: true}, nil, nil + } + + in := ci.RenderInputs.Installation + // The wasm is baked into the gateway envoy-proxy image. Resolve it with the same + // GetReference the base render uses for every image; the hook has the ImageSet here. + imageSet, err := imageset.GetImageSet(ctx, ci.Client, in.Variant) + if err != nil { + return wafRenderData{}, nil, err + } + wasmImage, err := components.GetReference(components.ComponentGatewayAPIEnvoyProxy, in.Registry, in.ImagePath, in.ImagePrefix, imageSet) + if err != nil { + return wafRenderData{}, nil, err + } + + webhookTLS, err := ci.CertificateManager.GetOrCreateKeyPair( + ci.Client, + applicationlayer.WAFWebhookServerTLSSecretName, + common.OperatorNamespace(), + dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, ci.RenderInputs.ClusterDomain), + ) + if err != nil { + return wafRenderData{}, nil, err + } + + pullSecrets, err := utils.GetInstallationPullSecrets(in, ci.Client) + if err != nil { + return wafRenderData{}, nil, err + } + var pullSecret *corev1.Secret + if len(pullSecrets) > 0 { + pullSecret, _ = MergeWAFPullSecret(pullSecrets) + } + + var caCert *corev1.ConfigMap + if ci.RenderInputs.TrustedBundle != nil { + caCert = ci.RenderInputs.TrustedBundle.ConfigMap(common.CalicoNamespace) + caCert.Name = WASMCACertName + } + + return wafRenderData{ + gatewayAPIPresent: true, + enabled: true, + wasmImage: wasmImage, + pullSecret: pullSecret, + caCert: caCert, + webhookTLS: webhookTLS, + caBundle: ci.CertificateManager.KeyPair().GetCertificatePEM(), + }, webhookTLS, nil +} + +// MergeWAFPullSecret synthesizes the dedicated WAF wasm pull secret +// (tigera-waf-pull-secret) by merging the registry auths of every Installation pull +// secret. The EnvoyExtensionPolicy image source takes a single pullSecretRef, so a +// merged secret is the only way to honor multiple Installation pull secrets for the +// Coraza wasm OCI pull (e.g. the Tigera pull secret plus a private registry mirror). +// +// If the same registry appears in more than one secret, the first secret in +// Installation order wins. Secrets that cannot be parsed are skipped and their names +// returned, so the caller can log them without failing the reconcile. Returns a nil +// Secret when no registry auths could be collected. +func MergeWAFPullSecret(pullSecrets []*corev1.Secret) (*corev1.Secret, []string) { + merged := map[string]json.RawMessage{} + var skipped []string + for _, s := range pullSecrets { + auths, err := registryAuths(s) + if err != nil { + skipped = append(skipped, s.Name) + continue + } + for registry, auth := range auths { + if _, ok := merged[registry]; !ok { + merged[registry] = auth + } + } + } + if len(merged) == 0 { + return nil, skipped + } + + // Marshalling a map sorts its keys, so the rendered bytes are deterministic and + // do not churn the object on every reconcile. + data, err := json.Marshal(map[string]map[string]json.RawMessage{"auths": merged}) + if err != nil { + // Each auth entry round-trips from a successful Unmarshal above, so this + // cannot fail in practice; treat it as nothing to render. + return nil, skipped + } + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: WASMPullSecretName, Namespace: common.CalicoNamespace}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: data}, + }, skipped +} + +// registryAuths extracts the per-registry auth entries from a pull secret of either +// the dockerconfigjson type (auths nested under an "auths" key) or the legacy +// dockercfg type (a bare registry -> auth map). +func registryAuths(s *corev1.Secret) (map[string]json.RawMessage, error) { + if raw, ok := s.Data[corev1.DockerConfigJsonKey]; ok { + var cfg struct { + Auths map[string]json.RawMessage `json:"auths"` + } + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, err + } + if len(cfg.Auths) == 0 { + return nil, fmt.Errorf("secret %s has no auths entries", s.Name) + } + return cfg.Auths, nil + } + if raw, ok := s.Data[corev1.DockerConfigKey]; ok { + var auths map[string]json.RawMessage + if err := json.Unmarshal(raw, &auths); err != nil { + return nil, err + } + if len(auths) == 0 { + return nil, fmt.Errorf("secret %s has no auths entries", s.Name) + } + return auths, nil + } + return nil, fmt.Errorf("secret %s has neither a %s nor a %s key", s.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) +} diff --git a/pkg/enterprise/installation/kubecontrollers_test.go b/pkg/enterprise/installation/kubecontrollers_test.go new file mode 100644 index 0000000000..8c7ca5ce38 --- /dev/null +++ b/pkg/enterprise/installation/kubecontrollers_test.go @@ -0,0 +1,495 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/enterprise" + "github.com/tigera/operator/pkg/enterprise/installation" + eoptions "github.com/tigera/operator/pkg/enterprise/options" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/applicationlayer" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/rbacmanagement" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls" +) + +var _ = Describe("kube-controllers enterprise modifier", func() { + // kubeControllersDeployment is a minimal stand-in for the calico-kube-controllers + // deployment the base render produces, so the modifier has something to mount onto. + kubeControllersDeployment := func() *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.KubeController, Namespace: common.CalicoNamespace}, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: kubecontrollers.KubeController}}, + }, + }, + }, + } + } + + It("mounts the metrics serving TLS keypair onto the deployment", func() { + eci, _, err := ext.Installation().ExtendInputs(ctx, newControllerInputs(operatorv1.CalicoEnterprise)) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + objs, _ := ext.Installation().Modify(extensionstest.KubeControllersStub{StubComponent: extensionstest.StubComponent{Create: []client.Object{kubeControllersDeployment()}, Delete: nil}, Cfg: nil}, ri).Objects() + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + + c := dp.Spec.Template.Spec.Containers[0] + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "TLS_KEY_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.key"}, + corev1.EnvVar{Name: "TLS_CRT_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.crt"}, + corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, + )) + Expect(c.VolumeMounts).To(ContainElement(HaveField("Name", kubecontrollers.KubeControllerPrometheusTLSSecret))) + Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", kubecontrollers.KubeControllerPrometheusTLSSecret))) + Expect(dp.Spec.Template.Annotations).NotTo(BeEmpty(), "expected the cert hash annotation") + }) + + It("adds the cert-management init container when certificate management is enabled", func() { + eci, _, err := ext.Installation().ExtendInputs(ctx, certManagementControllerInputs()) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + objs, _ := ext.Installation().Modify(extensionstest.KubeControllersStub{StubComponent: extensionstest.StubComponent{Create: []client.Object{kubeControllersDeployment()}, Delete: nil}, Cfg: nil}, ri).Objects() + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + + Expect(dp.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.InitContainers[0].Name).To(Equal(fmt.Sprintf("%s-key-cert-provisioner", kubecontrollers.KubeControllerPrometheusTLSSecret))) + }) +}) + +// certManagementControllerInputs builds a controller inputs whose certificate +// manager issues cert-management (CSR-based) keypairs. +func certManagementControllerInputs() controller.Inputs { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + ca, err := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) + Expect(err).NotTo(HaveOccurred()) + caCert, _, err := ca.Config.GetPEMBytes() + Expect(err).NotTo(HaveOccurred()) + + installation := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CertificateManagement: &operatorv1.CertificateManagement{CACert: caCert}, + } + certManager, err := certificatemanager.Create(c, installation, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: installation, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Client: c, + CertificateManager: certManager, + } +} + +var _ = Describe("calico-kube-controllers enterprise surface", func() { + calicoKubeControllersCfg := func(ci controller.Inputs) *kubecontrollers.KubeControllersConfiguration { + return &kubecontrollers.KubeControllersConfiguration{ + Installation: ci.RenderInputs.Installation, + ClusterDomain: ci.RenderInputs.ClusterDomain, + TrustedBundle: ci.RenderInputs.TrustedBundle, + MetricsPort: 9094, + Namespace: common.CalicoNamespace, + BindingNamespaces: []string{common.CalicoNamespace}, + } + } + + // render builds the base calico-kube-controllers objects and applies the + // enterprise modifier, exactly as the component handler does. + renderKubeControllers := func(ci controller.Inputs, ri render.Inputs) []client.Object { + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(ci)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + out, _ := ext.Installation().Modify(extensionstest.KubeControllersStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: nil}, ri).Objects() + return out + } + + kubeContainer := func(objs []client.Object) *corev1.Container { + dp, ok := extensions.FindObject[*appsv1.Deployment](objs, kubecontrollers.KubeController) + Expect(ok).To(BeTrue()) + return &dp.Spec.Template.Spec.Containers[0] + } + + It("layers the enterprise rules, controllers, and metrics TLS on (WAF off)", func() { + eci, _, err := ext.Installation().ExtendInputs(ctx, newControllerInputs(operatorv1.CalicoEnterprise)) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + objs := renderKubeControllers(newControllerInputs(operatorv1.CalicoEnterprise), ri) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage", + })) + // Metrics serving TLS wired from the keypair the hook created. + Expect(c.Env).To(ContainElement(HaveField("Name", "TLS_KEY_PATH"))) + // No GatewayAPI CR at all, so no WASM env, no WAF_GATEWAY_EXTENSION_ENABLED env, + // and no webhook objects. + Expect(c.Env).NotTo(ContainElement(HaveField("Name", "WASM_IMAGE"))) + Expect(c.Env).NotTo(ContainElement(HaveField("Name", "WAF_GATEWAY_EXTENSION_ENABLED"))) + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeFalse()) + }) + + It("layers the full WAF surface on when the GatewayAPI extension is enabled", func() { + ci := wafControllerInputs() + eci, managed, err := ext.Installation().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + names := []string{} + for _, kp := range managed { + names = append(names, kp.GetName()) + } + Expect(names).To(ContainElement(applicationlayer.WAFWebhookServerTLSSecretName)) + + objs := renderKubeControllers(ci, ri) + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("wafpolicies")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", + })) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, + })) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: installation.WASMPullSecretName})) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WASM_CA_CERT", Value: installation.WASMCACertName})) + Expect(c.Env).To(ContainElement(HaveField("Name", "WAF_WEBHOOK_CERT_DIR"))) + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WAF_GATEWAY_EXTENSION_ENABLED", Value: "true"})) + Expect(c.Ports).To(ContainElement(corev1.ContainerPort{Name: "waf-webhook", ContainerPort: int32(9443), Protocol: corev1.ProtocolTCP})) + + // The webhook surface, the wasm pull secret, and the wasm CA bundle are rendered. + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*corev1.Secret](objs, installation.WASMPullSecretName) + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*corev1.ConfigMap](objs, installation.WASMCACertName) + Expect(ok).To(BeTrue()) + }) + + It("deletes the WAF webhook surface when the extension is disabled", func() { + ci := newControllerInputs(operatorv1.CalicoEnterprise) + eci, _, err := ext.Installation().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(ci)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + _, toDelete := ext.Installation().Modify(extensionstest.KubeControllersStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: nil}, ri).Objects() + + _, ok := extensions.FindObject[*corev1.Service](toDelete, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeTrue(), "the webhook Service should be queued for deletion") + }) + + It("keeps the WAF controller wired but de-programs when GatewayAPI is present and WAF is disabled", func() { + ci := gatewayNoWAFControllerInputs() + eci, _, err := ext.Installation().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + objs := renderKubeControllers(ci, ri) + + // The applicationlayer controller and its WAF v3 RBAC stay wired even though WAF + // is disabled, so the controller can tear down the EnvoyExtensionPolicies it + // generated instead of losing its RBAC in the same reconcile (EV-6751). + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, kubecontrollers.KubeControllerRole) + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("wafpolicies")))) + + c := kubeContainer(objs) + Expect(c.Env).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", + })) + // It is told to de-program rather than program via WAF_GATEWAY_EXTENSION_ENABLED=false. + Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WAF_GATEWAY_EXTENSION_ENABLED", Value: "false"})) + + // But none of the active surface: no WASM env, no webhook port, and the webhook + // objects are not created. + Expect(c.Env).NotTo(ContainElement(HaveField("Name", "WASM_IMAGE"))) + Expect(c.Ports).NotTo(ContainElement(HaveField("Name", "waf-webhook"))) + _, ok = extensions.FindObject[*corev1.Service](objs, applicationlayer.WAFWebhookServiceName) + Expect(ok).To(BeFalse()) + }) + + It("adds the WAF webhook ingress rule to the network policy when enabled", func() { + ci := wafControllerInputs() + eci, _, err := ext.Installation().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllersPolicy(calicoKubeControllersCfg(ci), nil) + create, del := comp.Objects() + objs, _ := ext.Installation().Modify(extensionstest.KubeControllersPolicyStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: nil}, ri).Objects() + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](objs, kubecontrollers.KubeControllerNetworkPolicyName) + Expect(ok).To(BeTrue()) + Expect(policy.Spec.Ingress).To(ContainElement(v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), + }, + })) + }) + + // The base policy has no manager egress rule; reaching the manager is enterprise-only. + DescribeTable("adds the manager egress rule", + func(objs []client.Object, expected v3.EntityRule) { + ci := newControllerInputs(operatorv1.CalicoEnterprise, objs...) + eci, _, err := ext.Installation().ExtendInputs(ctx, ci) + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllersPolicy(calicoKubeControllersCfg(ci), nil) + create, del := comp.Objects() + out, _ := ext.Installation().Modify(extensionstest.KubeControllersPolicyStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: nil}, eci.RenderInputs).Objects() + + policy, ok := extensions.FindObject[*v3.NetworkPolicy](out, kubecontrollers.KubeControllerNetworkPolicyName) + Expect(ok).To(BeTrue()) + Expect(policy.Spec.Egress).To(ContainElement(v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: expected, + })) + }, + Entry("direct to the manager on a management or standalone cluster", + nil, networkpolicy.DefaultHelper().ManagerEntityRule()), + Entry("through Guardian on a managed cluster", + []client.Object{&operatorv1.ManagementClusterConnection{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}}}, + render.GuardianEntityRule), + ) + + Context("RBAC management gate", func() { + gate := func(value string) client.Object { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: rbacmanagement.ConfigMapName, Namespace: common.CalicoNamespace}, + Data: map[string]string{rbacmanagement.ConfigMapKey: value}, + } + } + + // renderWith runs the extension end to end and returns the created and deleted objects. + renderWith := func(s extensions.Extensions, objs ...client.Object) ([]client.Object, []client.Object) { + ci := newControllerInputs(operatorv1.CalicoEnterprise, objs...) + eci, _, err := s.Installation().ExtendInputs(ctx, ci) + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(ci)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + return s.Installation().Modify(extensionstest.KubeControllersStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: nil}, eci.RenderInputs).Objects() + } + + enabledControllers := func(objs []client.Object) string { + for _, env := range kubeContainer(objs).Env { + if env.Name == "ENABLED_CONTROLLERS" { + return env.Value + } + } + Fail("no ENABLED_CONTROLLERS env on the kube-controllers container") + return "" + } + + It("reads a missing ConfigMap as disabled", func() { + create, del := renderWith(ext) + Expect(enabledControllers(create)).NotTo(ContainSubstring("rbacsync")) + + _, ok := extensions.FindObject[*rbacv1.Role](del, "calico-kube-controllers-rbac-sync") + Expect(ok).To(BeTrue(), "the rbacsync Role should be queued for deletion") + }) + + It("follows the admin's value once they create the ConfigMap", func() { + create, _ := renderWith(ext, gate("true")) + Expect(enabledControllers(create)).To(ContainSubstring("rbacsync")) + + _, ok := extensions.FindObject[*rbacv1.Role](create, "calico-kube-controllers-rbac-sync") + Expect(ok).To(BeTrue()) + _, ok = extensions.FindObject[*rbacv1.RoleBinding](create, "calico-kube-controllers-rbac-sync") + Expect(ok).To(BeTrue()) + }) + + It("switches the feature back off when the admin sets the value to false", func() { + create, del := renderWith(ext, gate("false")) + Expect(enabledControllers(create)).NotTo(ContainSubstring("rbacsync")) + + _, ok := extensions.FindObject[*rbacv1.Role](del, "calico-kube-controllers-rbac-sync") + Expect(ok).To(BeTrue()) + }) + + It("withholds rbacsync on a multi-tenant management cluster even with the gate on", func() { + // Multi-tenant force-disables the feature on the ui-apis side. + multiTenant := enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{MultiTenant: true}) + create, _ := renderWith(multiTenant, gate("true")) + Expect(enabledControllers(create)).NotTo(ContainSubstring("rbacsync")) + }) + + It("never creates the ConfigMap itself", func() { + create, _ := renderWith(ext, gate("true")) + _, ok := extensions.FindObject[*corev1.ConfigMap](create, rbacmanagement.ConfigMapName) + Expect(ok).To(BeFalse(), "the gate is admin-owned, so the operator must not render it") + }) + }) + + It("binds kube-controllers to the managed-cluster watch role only on a management cluster", func() { + ci := newControllerInputs(operatorv1.CalicoEnterprise, + &operatorv1.ManagementCluster{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}}) + eci, _, err := ext.Installation().ExtendInputs(ctx, ci) + Expect(err).NotTo(HaveOccurred()) + + comp := kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(ci)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del := comp.Objects() + out, _ := ext.Installation().Modify(extensionstest.KubeControllersStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: nil}, eci.RenderInputs).Objects() + _, ok := extensions.FindObject[*rbacv1.ClusterRoleBinding](out, kubecontrollers.ManagedClustersWatchRoleBindingName) + Expect(ok).To(BeTrue()) + + // No ManagementCluster, no binding. + plain := newControllerInputs(operatorv1.CalicoEnterprise) + eci, _, err = ext.Installation().ExtendInputs(ctx, plain) + Expect(err).NotTo(HaveOccurred()) + comp = kubecontrollers.NewCalicoKubeControllers(calicoKubeControllersCfg(plain)) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + create, del = comp.Objects() + out, _ = ext.Installation().Modify(extensionstest.KubeControllersStub{StubComponent: extensionstest.StubComponent{Create: create, Delete: del}, Cfg: nil}, eci.RenderInputs).Objects() + _, ok = extensions.FindObject[*rbacv1.ClusterRoleBinding](out, kubecontrollers.ManagedClustersWatchRoleBindingName) + Expect(ok).To(BeFalse()) + }) +}) + +// wafControllerInputs builds a controller inputs with a WAF-enabled GatewayAPI CR +// and an install pull secret, so the installation hook produces the full WAF data. +func wafControllerInputs() controller.Inputs { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + Expect(c.Create(context.Background(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "pull", Namespace: common.OperatorNamespace()}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte(`{"auths":{"reg.example.com":{"auth":"abc"}}}`)}, + })).NotTo(HaveOccurred()) + + enabled := operatorv1.WAFExtensionStateEnabled + Expect(c.Create(context.Background(), &operatorv1.GatewayAPI{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: operatorv1.GatewayAPISpec{ + Extensions: &operatorv1.GatewayAPIExtensions{WAF: &operatorv1.WAFExtensionSpec{State: &enabled}}, + }, + })).NotTo(HaveOccurred()) + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "test-reg/", + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "pull"}}, + }, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Client: c, + CertificateManager: certManager, + } +} + +// gatewayNoWAFControllerInputs builds a controller inputs with a GatewayAPI CR +// present but its WAF extension explicitly disabled. The applicationlayer controller +// and its RBAC stay wired so it can de-program, but no active WAF surface is produced +// (EV-6751). +func gatewayNoWAFControllerInputs() controller.Inputs { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + c := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + Expect(c.Create(context.Background(), &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "pull", Namespace: common.OperatorNamespace()}, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte(`{"auths":{"reg.example.com":{"auth":"abc"}}}`)}, + })).NotTo(HaveOccurred()) + + disabled := operatorv1.WAFExtensionStateDisabled + Expect(c.Create(context.Background(), &operatorv1.GatewayAPI{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: operatorv1.GatewayAPISpec{ + Extensions: &operatorv1.GatewayAPIExtensions{WAF: &operatorv1.WAFExtensionSpec{State: &disabled}}, + }, + })).NotTo(HaveOccurred()) + + certManager, err := certificatemanager.Create(c, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "test-reg/", + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "pull"}}, + }, + FelixConfiguration: &v3.FelixConfiguration{}, + TrustedBundle: certManager.CreateTrustedBundle(), + ClusterDomain: "cluster.local", + }, + Client: c, + CertificateManager: certManager, + } +} diff --git a/pkg/enterprise/installation/node.go b/pkg/enterprise/installation/node.go new file mode 100644 index 0000000000..61bed69ff2 --- /dev/null +++ b/pkg/enterprise/installation/node.go @@ -0,0 +1,280 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation + +import ( + "fmt" + "slices" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" +) + +const ( + // defaultNodeReporterPort is the port calico/node reports Enterprise internal + // metrics on when FelixConfiguration does not override prometheusReporterPort. + defaultNodeReporterPort = 9081 + + // defaultFelixMetricsPort is the Felix prometheus metrics port used when + // FelixConfiguration does not override prometheusMetricsPort. + defaultFelixMetricsPort = 9091 +) + +// modifyNode layers Calico Enterprise behavior onto the rendered calico/node +// objects: the extra RBAC rules, the node-metrics Service, and the Enterprise +// daemonset configuration (flow/DNS log env, prometheus reporter, BGP metrics +// readiness check, multi-interface mode, and the calico log volume). +func modifyNode(ri render.Inputs, objs, del []client.Object) ([]client.Object, []client.Object) { + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName); ok { + role.Rules = append(role.Rules, nodeEnterpriseRules()...) + } + + // The Network resource is only available in Enterprise / Cloud at this time. + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoCNIPluginObjectName); ok { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"networks"}, + Verbs: []string{"get"}, + }) + } + + if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName); ok { + modifyNodeDaemonSet(ri, ds) + } + + return append(objs, nodeMetricsService(ri)), del +} + +// nodeEnterpriseRules are the additional cluster role rules calico/node needs in +// Calico Enterprise. +func nodeEnterpriseRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + // Calico Enterprise needs to be able to read additional resources. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "bfdconfigurations", + "egressgatewaypolicies", + "externalnetworks", + "licensekeys", + "networks", + "packetcaptures", + "remoteclusterconfigurations", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + // Tigera Secure updates status for packet captures. + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "packetcaptures", + "packetcaptures/status", + }, + Verbs: []string{"update"}, + }, + } +} + +// modifyNodeDaemonSet applies the Enterprise-specific daemonset changes that the +// base render leaves out: the Enterprise felix env, multi-interface mode, the +// BGP metrics readiness check, and the prometheus reporter keypair mount. The +// calico log volume is mounted by the base render for both variants, so it is +// not handled here. +func modifyNodeDaemonSet(ri render.Inputs, ds *appsv1.DaemonSet) { + spec := &ds.Spec.Template.Spec + + // Collecting process info for flow logs reads from the host's process table. + if installationData(ri).collectProcessPath { + spec.HostPID = true + } + + multiInterfaceMode := multiInterfaceModeEnv(ri.Installation) + + // MultiInterfaceMode is rejected in validation unless the CNI is Calico, which is + // also what gates the install-cni container. + if multiInterfaceMode != nil { + cni := render.MustContainer(spec, render.InstallCNIContainerName) + cni.Env = append(cni.Env, *multiInterfaceMode) + } + + c := render.MustContainer(spec, render.CalicoNodeObjectName) + c.Env = append(c.Env, nodeEnterpriseEnv(ri)...) + + // Add the BGP metrics readiness check, but only when the base render kept the + // bird readiness check (i.e. BGP is in use and we're not on VPP). + if c.ReadinessProbe != nil && c.ReadinessProbe.Exec != nil && slices.Contains(c.ReadinessProbe.Exec.Command, "--bird-ready") { + c.ReadinessProbe.Exec.Command = append(c.ReadinessProbe.Exec.Command, "--bgp-metrics-ready") + } + + mountNodePrometheusTLS(ri, ds) +} + +// mountNodePrometheusTLS mounts the node prometheus reporter keypair onto the +// daemonset: the volume, the calico-node volume mount, the cert-management init +// container (when in use), and the pod hash annotation that rolls the pods on +// cert rotation. The keypair has cluster side effects, so the enterprise setup +// creates it and hands it in via ri rather than the modifier building it. In +// core (calico) the keypair is never created, so the base node render carries +// no prometheus mount at all. +func mountNodePrometheusTLS(ri render.Inputs, ds *appsv1.DaemonSet) { + tls := installationData(ri).nodePrometheusTLS + if tls == nil || ri.TrustedBundle == nil { + return + } + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, tls.Volume()) + + c := render.MustContainer(spec, render.CalicoNodeObjectName) + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeLinux)) + if tls.UseCertificateManagement() { + spec.InitContainers = append(spec.InitContainers, tls.InitContainer(common.CalicoNamespace, c.SecurityContext)) + } + + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() +} + +// nodeEnterpriseEnv is the Enterprise felix configuration added to the +// calico/node container. +func nodeEnterpriseEnv(ri render.Inputs) []corev1.EnvVar { + data := installationData(ri) + env := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", NodeReporterPort(ri.FelixConfiguration))}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + if data.collectProcessPath { + env = append(env, corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"}) + } + + if mode := multiInterfaceModeEnv(ri.Installation); mode != nil { + env = append(env, *mode) + } + + tls := data.nodePrometheusTLS + if tls != nil && ri.TrustedBundle != nil { + env = append(env, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: ri.TrustedBundle.MountPath()}, + ) + } + + return env +} + +// multiInterfaceModeEnv returns the MULTI_INTERFACE_MODE env var when the +// installation configures it, or nil otherwise. +func multiInterfaceModeEnv(install *operatorv1.InstallationSpec) *corev1.EnvVar { + if install.CalicoNetwork != nil && install.CalicoNetwork.MultiInterfaceMode != nil { + return &corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: install.CalicoNetwork.MultiInterfaceMode.Value()} + } + return nil +} + +// nodeMetricsService builds the enterprise-only calico-node-metrics Service. +func nodeMetricsService(ri render.Inputs) *corev1.Service { + reporterPort := NodeReporterPort(ri.FelixConfiguration) + felixPort := felixMetricsPort(ri.FelixConfiguration) + felixEnabled := ri.FelixConfiguration != nil && utils.IsFelixPrometheusMetricsEnabled(ri.FelixConfiguration) + + ports := []corev1.ServicePort{ + { + Name: "calico-metrics-port", + Port: int32(reporterPort), + TargetPort: intstr.FromInt(reporterPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "calico-bgp-metrics-port", + Port: render.NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(render.NodeBGPReporterPort)), + Protocol: corev1.ProtocolTCP, + }, + } + if felixEnabled { + ports = append(ports, corev1.ServicePort{ + Name: "felix-metrics-port", + Port: int32(felixPort), + TargetPort: intstr.FromInt(felixPort), + Protocol: corev1.ProtocolTCP, + }) + } + + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.CalicoNodeMetricsService, + Namespace: common.CalicoNamespace, + Labels: map[string]string{"k8s-app": render.CalicoNodeObjectName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": render.CalicoNodeObjectName}, + ClusterIP: "None", + Ports: ports, + }, + } +} + +// ValidateReporterPort rejects the unsupported zero prometheus reporter port. +// The node and windows controller extensions share it. +func ValidateReporterPort(fc *v3.FelixConfiguration) error { + if fc != nil && fc.Spec.PrometheusReporterPort != nil && *fc.Spec.PrometheusReporterPort == 0 { + return extensions.InvalidConfigf("felixConfiguration prometheusReporterPort=0 not supported") + } + return nil +} + +// NodeReporterPort returns the reporter metrics port from the FelixConfiguration, +// falling back to the default. The node-metrics Service and the +// FELIX_PROMETHEUSREPORTERPORT env var both derive from here so they can't drift. +func NodeReporterPort(fc *v3.FelixConfiguration) int { + if fc != nil && fc.Spec.PrometheusReporterPort != nil { + return *fc.Spec.PrometheusReporterPort + } + return defaultNodeReporterPort +} + +// felixMetricsPort returns the Felix prometheus metrics port from the +// FelixConfiguration, falling back to the default. +func felixMetricsPort(fc *v3.FelixConfiguration) int { + if fc != nil && fc.Spec.PrometheusMetricsPort != nil { + return *fc.Spec.PrometheusMetricsPort + } + return defaultFelixMetricsPort +} diff --git a/pkg/enterprise/installation/node_enterprise_test.go b/pkg/enterprise/installation/node_enterprise_test.go new file mode 100644 index 0000000000..f4bf20b78f --- /dev/null +++ b/pkg/enterprise/installation/node_enterprise_test.go @@ -0,0 +1,257 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// getTyphaNodeTLS builds the node/typha TLS bundle the node render expects. +func getTyphaNodeTLS(cli client.Client, certificateManager certificatemanager.CertificateManager) *render.TyphaNodeTLS { + nodeKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaNonClusterHostKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName+render.TyphaNonClusterHostSuffix, common.OperatorNamespace(), []string{render.FelixCommonName + render.TyphaNonClusterHostSuffix}) + Expect(err).NotTo(HaveOccurred()) + + trustedBundle := certificateManager.CreateTrustedBundle(nodeKeyPair, typhaKeyPair) + + return &render.TyphaNodeTLS{ + TrustedBundle: trustedBundle, + TyphaSecret: typhaKeyPair, + TyphaSecretNonClusterHost: typhaNonClusterHostKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + } +} + +// These tests run the real node/typha render output through the registered +// enterprise modifiers. The render suite registers the enterprise extensions in +// its BeforeSuite, so this exercises the same integrated behavior the operator +// binary produces - and, importantly, catches a modifier whose FindObject stops +// matching because render renamed an object or container. +var _ = Describe("node enterprise modifier integration", func() { + var ( + cli client.Client + certManager certificatemanager.CertificateManager + typhaNodeTLS *render.TyphaNodeTLS + instance *operatorv1.InstallationSpec + renderInputs render.Inputs + nodePrometheusTLS certificatemanagement.KeyPairInterface + ) + + nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { + for i := range ds.Spec.Template.Spec.Containers { + if ds.Spec.Template.Spec.Containers[i].Name == render.CalicoNodeObjectName { + return &ds.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + BeforeEach(func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + var err error + certManager, err = certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + typhaNodeTLS = getTyphaNodeTLS(cli, certManager) + + nodePrometheusTLS, err = certManager.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics"}) + Expect(err).NotTo(HaveOccurred()) + typhaNodeTLS.TrustedBundle.AddCertificates(nodePrometheusTLS) + + confDir, binDir := render.DefaultCNIDirectories(operatorv1.ProviderNone) + bgp := operatorv1.BGPEnabled + instance = &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{ + Type: operatorv1.PluginCalico, + IPAM: &operatorv1.IPAMSpec{Type: operatorv1.IPAMPluginCalico}, + BinDir: &binDir, + ConfDir: &confDir, + }, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + BGP: &bgp, + IPPools: []operatorv1.IPPool{{CIDR: "192.168.1.0/16"}}, + }, + } + + // Build the render inputs the way the controller does: run the enterprise + // controller extension, which stashes the node prometheus keypair in the + // context for the node modifier to read. + ci := controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Client: cli, + CertificateManager: certManager, + } + eci, _, err := ext.Installation().ExtendInputs(ctx, ci) + Expect(err).NotTo(HaveOccurred()) + renderInputs = eci.RenderInputs + }) + + // renderNodeObjects renders the real node component and applies the registered + // modifier, exactly as the componentHandler does. + renderNodeObjects := func(ri render.Inputs) []client.Object { + cfg := &render.NodeConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + IPPools: instance.CalicoNetwork.IPPools, + } + comp := render.Node(cfg) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, _ := comp.Objects() + out, _ := ext.Installation().Modify(extensionstest.NodeStub{StubComponent: extensionstest.StubComponent{Create: objs, Delete: nil}, Cfg: nil}, ri).Objects() + return out + } + + It("appends the node metrics service to the real render output", func() { + objs := renderNodeObjects(renderInputs) + svc, ok := extensions.FindObject[*corev1.Service](objs, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue(), "expected the modifier to append %s", render.CalicoNodeMetricsService) + Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) + }) + + It("adds MULTI_INTERFACE_MODE to the real node and install-cni containers", func() { + mode := operatorv1.MultiInterfaceModeMultus + instance.CalicoNetwork.MultiInterfaceMode = &mode + + objs := renderNodeObjects(renderInputs) + ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + + want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} + Expect(nodeContainer(ds).Env).To(ContainElement(want)) + + cni, ok := render.Container(&ds.Spec.Template.Spec, render.InstallCNIContainerName) + Expect(ok).To(BeTrue()) + Expect(cni.Env).To(ContainElement(want)) + }) + + It("adds the enterprise rules to the real cluster roles", func() { + objs := renderNodeObjects(renderInputs) + + nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoNodeObjectName) + Expect(ok).To(BeTrue()) + Expect(nodeRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + cniRole, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.CalicoCNIPluginObjectName) + Expect(ok).To(BeTrue()) + Expect(cniRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("networks")))) + }) + + It("rewrites the real node daemonset for enterprise", func() { + objs := renderNodeObjects(renderInputs) + ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + + c := nodeContainer(ds) + Expect(c).NotTo(BeNil()) + + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + )) + // The reporter cert env is wired from the NodePrometheusTLS keypair the + // builder creates, and the modifier mounts that keypair onto the daemonset. + Expect(c.Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(nodePrometheusTLS.Volume())) + Expect(c.VolumeMounts).To(ContainElement(nodePrometheusTLS.VolumeMount(rmeta.OSTypeLinux))) + Expect(ds.Spec.Template.Annotations).To(HaveKey(nodePrometheusTLS.HashAnnotationKey())) + + // BGP is enabled, so the bird readiness check is present and the modifier + // adds the BGP metrics check. + Expect(c.ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) + }) + + It("enables process-path collection when the LogCollector requests it", func() { + enable := operatorv1.CollectProcessPathEnable + Expect(cli.Create(context.Background(), &operatorv1.LogCollector{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + Spec: operatorv1.LogCollectorSpec{CollectProcessPath: &enable}, + })).NotTo(HaveOccurred()) + + eci, _, err := ext.Installation().ExtendInputs(ctx, controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: instance, + TrustedBundle: typhaNodeTLS.TrustedBundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Client: cli, + CertificateManager: certManager, + }) + Expect(err).NotTo(HaveOccurred()) + ri := eci.RenderInputs + + ds, ok := extensions.FindObject[*appsv1.DaemonSet](renderNodeObjects(ri), common.NodeDaemonSetName) + Expect(ok).To(BeTrue()) + Expect(ds.Spec.Template.Spec.HostPID).To(BeTrue()) + Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"})) + }) + + It("adds the enterprise rules to the real typha cluster role", func() { + comp := render.Typha(&render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: instance, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + }) + Expect(comp.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, _ := comp.Objects() + objs, _ = ext.Installation().Modify(extensionstest.TyphaStub{StubComponent: extensionstest.StubComponent{Create: objs, Delete: nil}, Cfg: nil}, renderInputs).Objects() + + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, "calico-typha") + Expect(ok).To(BeTrue()) + Expect(role.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + }) +}) diff --git a/pkg/enterprise/installation/node_test.go b/pkg/enterprise/installation/node_test.go new file mode 100644 index 0000000000..ae2710155b --- /dev/null +++ b/pkg/enterprise/installation/node_test.go @@ -0,0 +1,184 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + client "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("node enterprise image override", func() { + It("selects the enterprise node image for the enterprise variant", func() { + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} + Expect(ext.Installation().Images().Resolve("node", components.ComponentCalicoNode, ent)).To(Equal(components.ComponentTigeraNode)) + }) + + It("leaves the default in place for the Calico variant", func() { + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + Expect(ext.Installation().Images().Resolve("node", components.ComponentCalicoNode, calico)).To(Equal(components.ComponentCalicoNode)) + }) +}) + +var _ = Describe("node enterprise modifier", func() { + // newObjs returns the subset of rendered node objects the modifier touches. + newObjs := func() []client.Object { + return []client.Object{ + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.CalicoNodeObjectName}}, + &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.CalicoCNIPluginObjectName}}, + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.NodeDaemonSetName}, + Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "install-cni"}}, + Containers: []corev1.Container{{ + Name: render.CalicoNodeObjectName, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{ + Command: []string{"/bin/calico-node", "-bird-ready", "--bird-ready", "--felix-ready"}, + }}}, + }}, + }}}, + }, + } + } + + nodeContainer := func(ds *appsv1.DaemonSet) *corev1.Container { + for i := range ds.Spec.Template.Spec.Containers { + if ds.Spec.Template.Spec.Containers[i].Name == render.CalicoNodeObjectName { + return &ds.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + entIn := func() render.Inputs { + return render.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise}} + } + + It("adds the enterprise cluster role rules", func() { + out, _ := ext.Installation().Modify(extensionstest.NodeStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, entIn()).Objects() + + nodeRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) + Expect(ok).To(BeTrue()) + Expect(nodeRole.Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + + cniRole, ok := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoCNIPluginObjectName) + Expect(ok).To(BeTrue()) + Expect(cniRole.Rules).To(ContainElement(HaveField("Resources", ConsistOf("networks")))) + }) + + It("adds the enterprise felix env to the node container", func() { + out, _ := ext.Installation().Modify(extensionstest.NodeStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, entIn()).Objects() + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + c := nodeContainer(ds) + + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + corev1.EnvVar{Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + )) + }) + + It("derives the reporter port from FelixConfiguration", func() { + reporter := 7081 + ctx := entIn() + ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{PrometheusReporterPort: &reporter}} + + out, _ := ext.Installation().Modify(extensionstest.NodeStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, ctx).Objects() + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(ds).Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "7081"})) + }) + + It("appends the BGP metrics readiness check when the bird check is present", func() { + out, _ := ext.Installation().Modify(extensionstest.NodeStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, entIn()).Objects() + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(ds).ReadinessProbe.Exec.Command).To(ContainElement("--bgp-metrics-ready")) + }) + + It("does not add the BGP metrics readiness check when the bird check is absent", func() { + objs := newObjs() + ds := objs[2].(*appsv1.DaemonSet) + ds.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec.Command = []string{"/bin/calico-node", "--felix-ready"} + + out, _ := ext.Installation().Modify(extensionstest.NodeStub{StubComponent: extensionstest.StubComponent{Create: objs, Delete: nil}, Cfg: nil}, entIn()).Objects() + got, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + Expect(nodeContainer(got).ReadinessProbe.Exec.Command).NotTo(ContainElement("--bgp-metrics-ready")) + }) + + It("adds MULTI_INTERFACE_MODE to the node and install-cni containers when configured", func() { + mode := operatorv1.MultiInterfaceModeMultus + ctx := entIn() + ctx.Installation.CalicoNetwork = &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &mode} + + out, _ := ext.Installation().Modify(extensionstest.NodeStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, ctx).Objects() + ds, _ := extensions.FindObject[*appsv1.DaemonSet](out, common.NodeDaemonSetName) + + want := corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: mode.Value()} + Expect(nodeContainer(ds).Env).To(ContainElement(want)) + Expect(ds.Spec.Template.Spec.InitContainers[0].Env).To(ContainElement(want)) + }) + + It("appends the node metrics service", func() { + out, _ := ext.Installation().Modify(extensionstest.NodeStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, entIn()).Objects() + svc, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeTrue()) + Expect(svc.Spec.Ports).To(HaveLen(2)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) + Expect(svc.Spec.Ports[1].Port).To(Equal(int32(9900))) + }) + + It("derives metrics service ports and felix-metrics-port from FelixConfiguration", func() { + reporter := 7081 + metrics := 7091 + enabled := true + ctx := entIn() + ctx.FelixConfiguration = &v3.FelixConfiguration{Spec: v3.FelixConfigurationSpec{ + PrometheusReporterPort: &reporter, + PrometheusMetricsPort: &metrics, + PrometheusMetricsEnabled: &enabled, + }} + + out, _ := ext.Installation().Modify(extensionstest.NodeStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, ctx).Objects() + svc, _ := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(svc.Spec.Ports).To(HaveLen(3)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(7081))) + Expect(svc.Spec.Ports[2].Name).To(Equal("felix-metrics-port")) + Expect(svc.Spec.Ports[2].Port).To(Equal(int32(7091))) + }) + + It("is a no-op when the operator runs as Calico", func() { + ctx := render.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out, _ := calicoExt.Installation().Modify(extensionstest.NodeStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, ctx).Objects() + + _, ok := extensions.FindObject[*corev1.Service](out, render.CalicoNodeMetricsService) + Expect(ok).To(BeFalse()) + nodeRole, _ := extensions.FindObject[*rbacv1.ClusterRole](out, render.CalicoNodeObjectName) + Expect(nodeRole.Rules).To(BeEmpty()) + }) +}) diff --git a/pkg/enterprise/installation/suite_test.go b/pkg/enterprise/installation/suite_test.go new file mode 100644 index 0000000000..4c29a26d3f --- /dev/null +++ b/pkg/enterprise/installation/suite_test.go @@ -0,0 +1,41 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" +) + +// calicoExt is the same Enterprise build running as Calico. +var ( + ext = enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{}) + calicoExt = enterprise.New(operatorv1.Calico, eoptions.Options{}) +) + +// ctx is the reconcile context the specs pass to the extension hooks. +var ctx = context.Background() + +func TestInstallation(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/installation Suite") +} diff --git a/pkg/enterprise/installation/typha.go b/pkg/enterprise/installation/typha.go new file mode 100644 index 0000000000..6f14aeeb04 --- /dev/null +++ b/pkg/enterprise/installation/typha.go @@ -0,0 +1,62 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation + +import ( + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/render" +) + +func modifyTypha(ri render.Inputs, objs, del []client.Object) ([]client.Object, []client.Object) { + if role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.TyphaClusterRoleName); ok { + role.Rules = append(role.Rules, rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{ + "bfdconfigurations", + "deeppacketinspections", + "egressgatewaypolicies", + "externalnetworks", + "licensekeys", + "networks", + "packetcaptures", + "remoteclusterconfigurations", + }, + Verbs: []string{"get", "list", "watch"}, + }) + } + + // Both Typha deployments need the interface mode, or the non-cluster-host one + // disagrees with the cluster one. + net := ri.Installation.CalicoNetwork + if net != nil && net.MultiInterfaceMode != nil { + for _, name := range []string{common.TyphaDeploymentName, common.TyphaDeploymentName + render.TyphaNonClusterHostSuffix} { + dep, ok := extensions.FindObject[*appsv1.Deployment](objs, name) + if !ok { + continue + } + + c := render.MustContainer(&dep.Spec.Template.Spec, render.TyphaContainerName) + c.Env = append(c.Env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: net.MultiInterfaceMode.Value()}) + } + } + + return objs, del +} diff --git a/pkg/enterprise/installation/typha_test.go b/pkg/enterprise/installation/typha_test.go new file mode 100644 index 0000000000..6a880d43b0 --- /dev/null +++ b/pkg/enterprise/installation/typha_test.go @@ -0,0 +1,122 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package installation_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" +) + +// These run the real typha render output through the registered modifier. A +// hand-built fixture would keep passing if render renamed what the modifier +// reaches for. +var _ = Describe("typha enterprise modifier", func() { + multiMode := operatorv1.MultiInterfaceModeMultus + + var typhaNodeTLS *render.TyphaNodeTLS + + BeforeEach(func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certManager, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + nodeKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + typhaKeyPair, err := certManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.TyphaCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaNodeTLS = &render.TyphaNodeTLS{ + TrustedBundle: certManager.CreateTrustedBundle(nodeKeyPair, typhaKeyPair), + TyphaSecret: typhaKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + } + }) + + // renderTypha renders the real typha component and runs the extension over it, + // exactly as the installation controller does. + renderTypha := func(r extensions.Extensions, install *operatorv1.InstallationSpec, ri render.Inputs) []client.Object { + component := render.Typha(&render.TyphaConfiguration{ + K8sServiceEp: k8sapi.ServiceEndpoint{}, + Installation: install, + TLS: typhaNodeTLS, + ClusterDomain: dns.DefaultClusterDomain, + FelixHealthPort: 9099, + }) + Expect(component.ResolveImages(nil)).NotTo(HaveOccurred()) + objs, del := component.Objects() + + out, _ := r.Installation().Modify(extensionstest.TyphaStub{StubComponent: extensionstest.StubComponent{Create: objs, Delete: del}, Cfg: nil}, ri).Objects() + return out + } + + typhaClusterRole := func(objs []client.Object) *rbacv1.ClusterRole { + role, ok := extensions.FindObject[*rbacv1.ClusterRole](objs, render.TyphaClusterRoleName) + Expect(ok).To(BeTrue()) + return role + } + + typhaContainer := func(objs []client.Object) *corev1.Container { + dep, ok := extensions.FindObject[*appsv1.Deployment](objs, common.TyphaDeploymentName) + Expect(ok).To(BeTrue()) + c, ok := render.Container(&dep.Spec.Template.Spec, render.TyphaContainerName) + Expect(ok).To(BeTrue()) + return c + } + + It("adds enterprise RBAC and MULTI_INTERFACE_MODE for the enterprise variant", func() { + install := &operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + CNI: &operatorv1.CNISpec{Type: operatorv1.PluginCalico}, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, + } + objs := renderTypha(ext, install, render.Inputs{Installation: install}) + + Expect(typhaClusterRole(objs).Rules).To(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + Expect(typhaContainer(objs).Env).To(ContainElement(corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: multiMode.Value()})) + }) + + It("is a no-op when the operator runs as Calico", func() { + install := &operatorv1.InstallationSpec{ + Variant: operatorv1.Calico, + CNI: &operatorv1.CNISpec{Type: operatorv1.PluginCalico}, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{MultiInterfaceMode: &multiMode}, + } + objs := renderTypha(calicoExt, install, render.Inputs{Installation: install}) + + Expect(typhaClusterRole(objs).Rules).NotTo(ContainElement(HaveField("Resources", ContainElement("licensekeys")))) + Expect(typhaContainer(objs).Env).NotTo(ContainElement(HaveField("Name", "MULTI_INTERFACE_MODE"))) + }) +}) diff --git a/pkg/render/kubecontrollers/waf_pull_secret_test.go b/pkg/enterprise/installation/waf_pull_secret_test.go similarity index 88% rename from pkg/render/kubecontrollers/waf_pull_secret_test.go rename to pkg/enterprise/installation/waf_pull_secret_test.go index 793374f169..6cb54f5f51 100644 --- a/pkg/render/kubecontrollers/waf_pull_secret_test.go +++ b/pkg/enterprise/installation/waf_pull_secret_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kubecontrollers_test +package installation_test import ( "encoding/json" @@ -22,7 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/enterprise/installation" ) func dockerConfigJSONSecret(name string, auths map[string]any) *corev1.Secret { @@ -49,7 +49,7 @@ func mergedAuths(t *testing.T, s *corev1.Secret) map[string]map[string]string { } func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("tigera-pull-secret", map[string]any{"quay.io": map[string]string{"auth": "dGlnZXJh"}}), dockerConfigJSONSecret("mirror-pull-secret", map[string]any{"registry.example.com": map[string]string{"auth": "bWlycm9y"}}), }) @@ -59,7 +59,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { if merged == nil { t.Fatal("expected a merged secret") } - if merged.Name != kubecontrollers.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { + if merged.Name != installation.WASMPullSecretName || merged.Namespace != common.CalicoNamespace { t.Fatalf("unexpected name/namespace: %s/%s", merged.Namespace, merged.Name) } if merged.Type != corev1.SecretTypeDockerConfigJson { @@ -72,7 +72,7 @@ func TestMergeWAFPullSecret_MergesDisjointRegistries(t *testing.T) { } func TestMergeWAFPullSecret_FirstSecretWinsOnDuplicateRegistry(t *testing.T) { - merged, _ := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, _ := installation.MergeWAFPullSecret([]*corev1.Secret{ dockerConfigJSONSecret("first", map[string]any{"quay.io": map[string]string{"auth": "Zmlyc3Q="}}), dockerConfigJSONSecret("second", map[string]any{"quay.io": map[string]string{"auth": "c2Vjb25k"}}), }) @@ -88,7 +88,7 @@ func TestMergeWAFPullSecret_SkipsUnparseableSecrets(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{ + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{ bad, dockerConfigJSONSecret("good", map[string]any{"quay.io": map[string]string{"auth": "Z29vZA=="}}), }) @@ -111,7 +111,7 @@ func TestMergeWAFPullSecret_LegacyDockercfg(t *testing.T) { Type: corev1.SecretTypeDockercfg, Data: map[string][]byte{corev1.DockerConfigKey: cfg}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{legacy}) + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{legacy}) if len(skipped) != 0 { t.Fatalf("expected no skipped secrets, got %v", skipped) } @@ -127,7 +127,7 @@ func TestMergeWAFPullSecret_NothingUsableReturnsNil(t *testing.T) { Type: corev1.SecretTypeDockerConfigJson, Data: map[string][]byte{corev1.DockerConfigJsonKey: []byte("not-json")}, } - merged, skipped := kubecontrollers.MergeWAFPullSecret([]*corev1.Secret{bad}) + merged, skipped := installation.MergeWAFPullSecret([]*corev1.Secret{bad}) if merged != nil { t.Fatalf("expected nil secret, got %v", merged) } @@ -141,8 +141,8 @@ func TestMergeWAFPullSecret_DeterministicOutput(t *testing.T) { dockerConfigJSONSecret("a", map[string]any{"z.example.com": map[string]string{"auth": "eg=="}, "a.example.com": map[string]string{"auth": "YQ=="}}), dockerConfigJSONSecret("b", map[string]any{"m.example.com": map[string]string{"auth": "bQ=="}}), } - first, _ := kubecontrollers.MergeWAFPullSecret(in) - second, _ := kubecontrollers.MergeWAFPullSecret(in) + first, _ := installation.MergeWAFPullSecret(in) + second, _ := installation.MergeWAFPullSecret(in) if string(first.Data[corev1.DockerConfigJsonKey]) != string(second.Data[corev1.DockerConfigJsonKey]) { t.Fatal("merged secret bytes must be deterministic across reconciles") } diff --git a/pkg/enterprise/kubecontrollers/es_kube_controllers_test.go b/pkg/enterprise/kubecontrollers/es_kube_controllers_test.go new file mode 100644 index 0000000000..5c682fc99e --- /dev/null +++ b/pkg/enterprise/kubecontrollers/es_kube_controllers_test.go @@ -0,0 +1,369 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubecontrollers + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + rtest "github.com/tigera/operator/pkg/render/common/test" + rkc "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/testutils" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var _ = Describe("es-kube-controllers rendering tests", func() { + var ( + instance *operatorv1.InstallationSpec + k8sServiceEp k8sapi.ServiceEndpoint + cfg rkc.KubeControllersConfiguration + cli client.Client + ) + + esEnvs := []corev1.EnvVar{ + {Name: "ELASTIC_HOST", Value: "tigera-secure-es-gateway-http.tigera-elasticsearch.svc"}, + {Name: "ELASTIC_PORT", Value: "9200", ValueFrom: nil}, + { + Name: "ELASTIC_USERNAME", Value: "", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ee-kube-controllers-elasticsearch-access", + }, + Key: "username", + }, + }, + }, + { + Name: "ELASTIC_PASSWORD", Value: "", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ee-kube-controllers-elasticsearch-access", + }, + Key: "password", + }, + }, + }, + {Name: "ELASTIC_CA", Value: certificatemanagement.TrustedCertBundleMountPath}, + } + + // The es-kube-controllers policy fixtures live next to the render package's + // testutils, so reference them relative to this enterprise subpackage. + expectedESPolicy := testutils.GetExpectedPolicyFromFile("../../render/testutils/expected_policies/es-kubecontrollers.json") + expectedESPolicyForOpenshift := testutils.GetExpectedPolicyFromFile("../../render/testutils/expected_policies/es-kubecontrollers_ocp.json") + + BeforeEach(func() { + // Initialize a default instance to use. Each test can override this to its + // desired configuration. + + miMode := operatorv1.MultiInterfaceModeNone + instance = &operatorv1.InstallationSpec{ + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + IPPools: []operatorv1.IPPool{{CIDR: "192.168.1.0/16"}}, + MultiInterfaceMode: &miMode, + }, + Registry: "test-reg/", + } + k8sServiceEp = k8sapi.ServiceEndpoint{} + + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + cfg = rkc.KubeControllersConfiguration{ + K8sServiceEp: k8sServiceEp, + Installation: instance, + ClusterDomain: dns.DefaultClusterDomain, + MetricsPort: 9094, + TrustedBundle: certificateManager.CreateTrustedBundle(), + Namespace: common.CalicoNamespace, + BindingNamespaces: []string{common.CalicoNamespace}, + } + }) + + It("should render all es-calico-kube-controllers resources for a default configuration (standalone) using CalicoEnterprise when logstorage and secrets exist", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, + {name: EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + } + + instance.Variant = operatorv1.CalicoEnterprise + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + + component := NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The Deployment should have the correct configuration. + dp := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + + Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) + envs := dp.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", Value: "authorization,elasticsearchconfiguration", + })) + Expect(envs).To(ContainElements(esEnvs)) + + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) + + Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) + + clusterRole := rtest.GetResource(resources, EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + })) + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + })) + }) + + It("should render all es-calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, + {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, + {name: EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, + {name: EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: rkc.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, + {name: EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, + {name: ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, + {name: EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, + } + + // Override configuration to match expected Enterprise config. + instance.Variant = operatorv1.CalicoEnterprise + cfg.ManagementCluster = &operatorv1.ManagementCluster{} + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + + component := NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The Deployment should have the correct configuration. + dp := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + + envs := dp.Spec.Template.Spec.Containers[0].Env + Expect(envs).To(ContainElement(corev1.EnvVar{ + Name: "ENABLED_CONTROLLERS", + Value: "authorization,elasticsearchconfiguration,managedcluster", + })) + + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) + + Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) + Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) + + Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) + + clusterRole := rtest.GetResource(resources, EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) + Expect(clusterRole.Rules).To(HaveLen(26), "cluster role should have 26 rules") + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + })) + Expect(clusterRole.Rules).To(ContainElement( + rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + })) + roleBindingWatch := rtest.GetResource(resources, rkc.ManagedClustersWatchRoleBindingName, "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) + Expect(roleBindingWatch.RoleRef.Name).To(Equal(render.ManagedClustersWatchClusterRoleName)) + Expect(roleBindingWatch.Subjects).To(ConsistOf([]rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: rkc.KubeControllerServiceAccount, + Namespace: common.CalicoNamespace, + }, + })) + }) + + It("should add the OIDC prefix env variables", func() { + instance.Variant = operatorv1.CalicoEnterprise + cfg.ManagementCluster = &operatorv1.ManagementCluster{} + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + cfg.Authentication = &operatorv1.Authentication{Spec: operatorv1.AuthenticationSpec{ + UsernamePrefix: "uOIDC:", + GroupsPrefix: "gOIDC:", + Openshift: &operatorv1.AuthenticationOpenshift{IssuerURL: "https://api.example.com"}, + }} + + component := NewElasticsearchKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + + depResource := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + Expect(depResource).ToNot(BeNil()) + deployment := depResource.(*appsv1.Deployment) + + var usernamePrefix, groupPrefix string + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == EsKubeController { + for _, env := range container.Env { + switch env.Name { + case "OIDC_AUTH_USERNAME_PREFIX": + usernamePrefix = env.Value + case "OIDC_AUTH_GROUP_PREFIX": + groupPrefix = env.Value + } + } + } + } + + Expect(usernamePrefix).To(Equal("uOIDC:")) + Expect(groupPrefix).To(Equal("gOIDC:")) + }) + + When("enableESOIDCWorkaround is true", func() { + It("should set the ENABLE_ELASTICSEARCH_OIDC_WORKAROUND env variable to true", func() { + instance.Variant = operatorv1.CalicoEnterprise + cfg.ManagementCluster = &operatorv1.ManagementCluster{} + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + cfg.MetricsPort = 9094 + component := NewElasticsearchKubeControllers(&cfg) + resources, _ := component.Objects() + + depResource := rtest.GetResource(resources, EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") + Expect(depResource).ToNot(BeNil()) + deployment := depResource.(*appsv1.Deployment) + + var esLicenseType string + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == EsKubeController { + for _, env := range container.Env { + if env.Name == "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND" { + esLicenseType = env.Value + } + } + } + } + + Expect(esLicenseType).To(Equal("true")) + }) + }) + + Context("es-kube-controllers calico-system rendering", func() { + policyName := types.NamespacedName{Name: "calico-system.es-kube-controller-access", Namespace: common.CalicoNamespace} + + getExpectedPolicy := func(scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { + if scenario.ManagedCluster { + return nil + } + + return testutils.SelectPolicyByProvider(scenario, expectedESPolicy, expectedESPolicyForOpenshift) + } + + DescribeTable("should render calico-system policy", + func(scenario testutils.CalicoSystemScenario) { + if scenario.OpenShift { + cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift + } else { + cfg.Installation.KubernetesProvider = operatorv1.ProviderNone + } + if scenario.ManagedCluster { + cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} + } else { + cfg.ManagementClusterConnection = nil + } + instance.Variant = operatorv1.CalicoEnterprise + cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret + + component := NewElasticsearchKubeControllers(&cfg) + resources, _ := component.Objects() + + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + expectedPolicy := getExpectedPolicy(scenario) + Expect(policy).To(Equal(expectedPolicy)) + }, + Entry("for management/standalone, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: false}), + Entry("for management/standalone, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: true}), + Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), + Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), + ) + }) +}) diff --git a/pkg/enterprise/kubecontrollers/kubecontrollers.go b/pkg/enterprise/kubecontrollers/kubecontrollers.go new file mode 100644 index 0000000000..6f1bad8149 --- /dev/null +++ b/pkg/enterprise/kubecontrollers/kubecontrollers.go @@ -0,0 +1,350 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package kubecontrollers holds the enterprise es-calico-kube-controllers assembly +// (a distinct deployment the logstorage controller reconciles) and the enterprise +// kube-controllers cluster role rules shared with the calico-kube-controllers +// modifier in pkg/enterprise/installation. +package kubecontrollers + +import ( + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + "github.com/tigera/operator/pkg/render" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + rkc "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/url" +) + +const ( + EsKubeController = "es-calico-kube-controllers" + EsKubeControllerRole = "es-calico-kube-controllers" + EsKubeControllerRoleBinding = "es-calico-kube-controllers" + EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" + EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + + ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" + ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" + ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" + ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" +) + +// NewElasticsearchKubeControllers fills the generic kube-controllers configuration +// for the enterprise es-calico-kube-controllers deployment and returns the rendered +// component. es-kube-controllers is a distinct deployment (talks to Elasticsearch via +// es-gateway) reconciled by the logstorage kube-controllers controller, so it's +// assembled here rather than through the render-time modifier mechanism. +func NewElasticsearchKubeControllers(cfg *rkc.KubeControllersConfiguration) render.Component { + cfg.Name = EsKubeController + cfg.ConfigName = "elasticsearch" + cfg.RoleName = EsKubeControllerRole + cfg.RoleBindingName = EsKubeControllerRoleBinding + cfg.MetricsName = EsKubeControllerMetrics + cfg.DisableConfigAPI = cfg.Tenant.MultiTenant() + + cfg.Rules = rkc.KubeControllersRoleCommonRules(cfg) + cfg.Rules = append(cfg.Rules, KubeControllersEnterpriseCommonRules(false, cfg.ManagementClusterConnection != nil)...) + // Calico Cloud's es-kube-controllers provisions RBAC for managed-cluster access, so it needs to + // create and update cluster roles and bindings. Enterprise only reads them. + clusterRoleVerbs := []string{"watch", "list", "get"} + if cfg.Cloud { + clusterRoleVerbs = append(clusterRoleVerbs, "create", "update") + } + + cfg.Rules = append(cfg.Rules, + rbacv1.PolicyRule{ + APIGroups: []string{"elasticsearch.k8s.elastic.co"}, + Resources: []string{"elasticsearches"}, + Verbs: []string{"watch", "get", "list"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"rbac.authorization.k8s.io"}, + Resources: []string{"clusterroles", "clusterrolebindings"}, + Verbs: clusterRoleVerbs, + }, + ) + + if !cfg.Tenant.MultiTenant() { + // Zero and single tenant clusters need elasticsearch configuration. + cfg.EnabledControllers = append(cfg.EnabledControllers, "authorization", "elasticsearchconfiguration") + if cfg.ManagementCluster != nil && cfg.Tenant == nil { + cfg.ManagedClusterWatchBinding = true + // Enterprise requires the managedcluster controller to push licenses. + cfg.EnabledControllers = append(cfg.EnabledControllers, "managedcluster") + } + } + + cfg.NetworkPolicy = esKubeControllersCalicoSystemPolicy(cfg) + cfg.DeprecatedNetworkPolicyName = "es-kube-controller-access" + cfg.ExtraEnv = esKubeControllersEnv(cfg) + + return rkc.NewKubeControllers(cfg) +} + +// esKubeControllersEnv builds the enterprise env vars for es-calico-kube-controllers. +func esKubeControllersEnv(cfg *rkc.KubeControllersConfiguration) []corev1.EnvVar { + var env []corev1.EnvVar + + if cfg.Tenant != nil { + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.Tenant.Spec.ID}) + } else if cfg.TenantID != "" { + // Calico Cloud reads the tenant from its cloud config rather than a Tenant CR. + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: cfg.TenantID}) + } + + // What started as a workaround is now the default behaviour. This feature uses our backend in order to + // log into Kibana for users from external identity providers, rather than configuring an authn realm + // in the Elastic stack. + env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) + if cfg.Authentication != nil { + env = append(env, + corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: cfg.Authentication.Spec.UsernamePrefix}, + corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: cfg.Authentication.Spec.GroupsPrefix}, + ) + } + + if cfg.TrustedBundle != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: cfg.TrustedBundle.MountPath()}) + } + if cfg.Installation.CalicoNetwork != nil && cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { + env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) + } + + if !cfg.Tenant.MultiTenant() { + _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(rmeta.OSTypeLinux, cfg.ClusterDomain, render.ElasticsearchNamespace)) + env = append(env, + relasticsearch.ElasticHostEnvVar(esHost), + relasticsearch.ElasticPortEnvVar(esPort), + relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), + relasticsearch.ElasticCAEnvVar(rmeta.OSTypeLinux), + ) + } + + return env +} + +// KubeControllersEnterpriseCommonRules are the Calico Enterprise cluster role rules +// shared by calico-kube-controllers and es-calico-kube-controllers. gatewayAPIPresent +// adds the WAF v3 (Gateway API add-on) rules - gated on the GatewayAPI CR existing, not +// on waf.state == Enabled, so the applicationlayer controller keeps the RBAC it needs +// to delete the EnvoyExtensionPolicies it generated while WAF is disabled (EV-6751); the +// rule set is identical enabled vs disabled, so toggling waf.state causes no ClusterRole +// churn. managedCluster adds the license-push rule a managed cluster's kube-controllers +// needs. +func KubeControllersEnterpriseCommonRules(gatewayAPIPresent, managedCluster bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, + }, + { + // The Federated Services Controller needs access to the remote kubeconfig secret + // in order to create a remote syncer. + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"watch", "list", "get"}, + }, + { + // Needed to validate the license + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + // Needed to update the status of the LicenseKey with the result of license validation. + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"licensekeys/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections"}, + Verbs: []string{"get", "watch", "list"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"deeppacketinspections/status"}, + Verbs: []string{"update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures"}, + Verbs: []string{"get", "list", "update"}, + }, + { + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"packetcaptures/status"}, + Verbs: []string{"update"}, + }, + } + + if gatewayAPIPresent { + rules = append(rules, wafRules()...) + } + + if managedCluster { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, + Resources: []string{"licensekeys"}, + Verbs: []string{"get", "create", "update", "list", "watch"}, + }, + ) + } + + return rules +} + +// wafRules are the WAF v3 (Gateway API add-on) cluster role rules, gated by +// GatewayAPI.spec.extensions.waf.state == Enabled. +func wafRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + // Application-layer (gateway-addons) reconcilers reconcile WAF resources + // against Gateway API targetRefs and emit events on the policy objects. + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies", "globalwafpolicies", + "wafplugins", "globalwafplugins", + "wafvalidationpolicies", "globalwafvalidationpolicies", + }, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/status", "globalwafpolicies/status", + "wafplugins/status", "globalwafplugins/status", + "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", + }, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{"applicationlayer.projectcalico.org"}, + Resources: []string{ + "wafpolicies/finalizers", "globalwafpolicies/finalizers", + "wafplugins/finalizers", "globalwafplugins/finalizers", + "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", + }, + Verbs: []string{"update"}, + }, + { + // Validate Gateway API targetRefs and surface attachment status. + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, + Verbs: []string{"get", "list", "watch", "update", "patch"}, + }, + { + APIGroups: []string{"gateway.networking.k8s.io"}, + Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + // controller-runtime Reconcilers (e.g. the applicationlayer manager) record + // events on watched objects via Recorder.Eventf; both core and events.k8s.io + // API groups are emitted depending on the kubernetes version. + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + { + APIGroups: []string{"events.k8s.io"}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch"}, + }, + // Application-layer reconciler replicates the WAF wasm pull Secret from + // the controller namespace (calico-system) into each WAFPolicy's + // namespace so the rendered EnvoyExtensionPolicy can reference it. Also + // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. + { + APIGroups: []string{""}, + Resources: []string{"secrets", "configmaps"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF + // targetRef to bind the Coraza wasm filter at the gateway / route. + { + APIGroups: []string{"gateway.envoyproxy.io"}, + Resources: []string{"envoyextensionpolicies"}, + Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, + }, + // Application-layer reconciler stamps each namespace with its allocated WAF + // rule-id range (applicationlayer.projectcalico.org/waf-id-range annotation) + // so application operators can author in-range rules. The base role already + // grants namespaces get/list/watch; the annotation write needs patch/update. + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "patch", "update"}, + }, + } +} + +func esKubeControllersCalicoSystemPolicy(cfg *rkc.KubeControllersConfiguration) *v3.NetworkPolicy { + if cfg.ManagementClusterConnection != nil { + return nil + } + + egressRules := []v3.Rule{} + egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: networkpolicy.Ports(443, 6443, 12388), + }, + }, + }...) + + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), + }, + }...) + + networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) + egressRules = append(egressRules, []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: networkpolicyHelper.ManagerEntityRule(), + }, + }...) + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: EsKubeControllerNetworkPolicyName, + Namespace: cfg.Namespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(EsKubeController), + Types: []v3.PolicyType{v3.PolicyTypeEgress}, + Egress: egressRules, + }, + } +} diff --git a/pkg/enterprise/kubecontrollers/suite_test.go b/pkg/enterprise/kubecontrollers/suite_test.go new file mode 100644 index 0000000000..a28166fba6 --- /dev/null +++ b/pkg/enterprise/kubecontrollers/suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubecontrollers_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestKubeControllers(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/kubecontrollers Suite") +} diff --git a/pkg/enterprise/options/options.go b/pkg/enterprise/options/options.go new file mode 100644 index 0000000000..bcf6fff145 --- /dev/null +++ b/pkg/enterprise/options/options.go @@ -0,0 +1,31 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +// Options is the Calico Enterprise controller-phase options, held by the hooks that +// need them. It lives in its own leaf package so the hooks and main can both +// reference it. +type Options struct { + // MultiTenant reports whether the operator runs in multi-tenant mode. + MultiTenant bool + + // Cloud reports whether this is a Calico Cloud install. + Cloud bool + + // ManageCRDs and UseV3CRDs mirror the operator's CRD management options. The + // installation hook watches the CRDs the variant adds. + ManageCRDs bool + UseV3CRDs bool +} diff --git a/pkg/enterprise/register.go b/pkg/enterprise/register.go new file mode 100644 index 0000000000..a75a2d87e2 --- /dev/null +++ b/pkg/enterprise/register.go @@ -0,0 +1,44 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package enterprise + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise/apiserver" + "github.com/tigera/operator/pkg/enterprise/clusterconnection" + "github.com/tigera/operator/pkg/enterprise/installation" + eoptions "github.com/tigera/operator/pkg/enterprise/options" + "github.com/tigera/operator/pkg/enterprise/windows" + "github.com/tigera/operator/pkg/extensions" +) + +// New builds the Calico Enterprise extensions. After the monorepo split this is what +// calico-private's main constructs instead. +func New(variant operatorv1.ProductVariant, o eoptions.Options) extensions.Extensions { + switch variant { + case operatorv1.CalicoEnterprise: + return extensions.New(extensions.Set{ + Installation: installation.New(variant, o), + Windows: windows.New(variant), + APIServer: apiserver.New(variant, o), + ClusterConnection: clusterconnection.New(variant), + }) + case operatorv1.Calico: + // Clean up what a prior Enterprise installation left behind. + return extensions.New(extensions.Set{APIServer: apiserver.CalicoCleanup{}}) + } + + return extensions.Extensions{} +} diff --git a/pkg/enterprise/windows/extension.go b/pkg/enterprise/windows/extension.go new file mode 100644 index 0000000000..386cc02327 --- /dev/null +++ b/pkg/enterprise/windows/extension.go @@ -0,0 +1,263 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/enterprise/installation" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/monitor" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Extension is the Calico Enterprise behavior for the windows controller. +type Extension struct { + variant operatorv1.ProductVariant + images *imageoverride.Overrides +} + +var _ extensions.WindowsExtension = &Extension{} + +// New returns the windows extension for the variant the operator resolved. +func New(variant operatorv1.ProductVariant) *Extension { + images := imageoverride.New() + images.Register(variant, render.ComponentNameWindowsNodeImg, components.ComponentTigeraNodeWindows) + images.Register(variant, render.ComponentNameWindowsCNIImg, components.ComponentTigeraCNIWindows) + + return &Extension{variant: variant, images: images} +} + +func (e *Extension) Images() *imageoverride.Overrides { + return e.images +} + +// Modify dispatches over the components the windows controller renders. +func (e *Extension) Modify(c render.Component, ri render.Inputs) render.Component { + switch c.(type) { + case render.WindowsComponent: + return extensions.Decorate(c, ri, e.variant, func(objs, del []client.Object) ([]client.Object, []client.Object) { + return modifyWindows(ri, objs, del) + }) + default: + return c + } +} + +// windowsRenderData is the controller-produced data the windows extension hands to +// its modifier through Inputs.Extension. +type windowsRenderData struct { + prometheusServerTLS certificatemanagement.KeyPairInterface +} + +// windowsData pulls the windows extension's render data back out of the render +// context, returning the zero value when none is set. +func windowsData(ri render.Inputs) windowsRenderData { + return render.ExtractExtensionData[windowsRenderData](ri) +} + +// Validate rejects windows installation config Calico Enterprise does not support. +// Watches registers the enterprise secrets the windows controller reconciles on. +func (e *Extension) Watches(c ctrlruntime.Controller) error { + for _, ns := range []string{common.CalicoNamespace, common.OperatorNamespace()} { + if err := utils.AddSecretsWatch(c, render.NodePrometheusTLSServerSecret, ns); err != nil { + return err + } + if err := utils.AddSecretsWatch(c, monitor.PrometheusClientTLSSecretName, ns); err != nil { + return err + } + } + return nil +} + +// ExtendInputs fetches the node prometheus keypair the installation controller +// created and stashes it in the render inputs for the windows modifier. +func (e *Extension) ExtendInputs(ctx context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) { + if err := installation.ValidateReporterPort(ci.RenderInputs.FelixConfiguration); err != nil { + return ci, nil, err + } + + tls, err := ci.CertificateManager.GetKeyPair( + ci.Client, + render.NodePrometheusTLSServerSecret, + common.OperatorNamespace(), + dns.GetServiceDNSNames(render.WindowsNodeMetricsService, common.CalicoNamespace, ci.RenderInputs.ClusterDomain), + ) + if err != nil { + return ci, nil, fmt.Errorf("error getting node prometheus TLS certificate: %w", err) + } + ci.RenderInputs.Extension = windowsRenderData{prometheusServerTLS: tls} + return ci, nil, nil +} + +// modifyWindows layers Calico Enterprise behavior onto the rendered +// calico-node-windows objects: the node-metrics Service and the Enterprise +// daemonset configuration (flow/DNS log env, prometheus reporter, trusted DNS +// servers, the calico log volume, and the prometheus reporter keypair mount). +func modifyWindows(ri render.Inputs, objs, del []client.Object) ([]client.Object, []client.Object) { + if ds, ok := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName); ok { + modifyWindowsDaemonSet(ri, ds) + } + + return append(objs, windowsNodeMetricsService(ri)), del +} + +// windowsNodeContainers are the containers the enterprise layering applies to. +// Confd only renders when BGP is enabled. +func windowsNodeContainers(spec *corev1.PodSpec) []*corev1.Container { + cs := render.MustContainers(spec, render.WindowsNodeContainerNames...) + if confd, ok := render.Container(spec, render.WindowsConfdContainerName); ok { + cs = append(cs, confd) + } + return cs +} + +func modifyWindowsDaemonSet(ri render.Inputs, ds *appsv1.DaemonSet) { + dirOrCreate := corev1.HostPathDirectoryOrCreate + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, corev1.Volume{ + Name: "var-log-calico", + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, + }) + + for _, c := range windowsNodeContainers(spec) { + c.Env = append(c.Env, windowsEnterpriseEnv(ri)...) + + // Enterprise mounts the calico log directory in place of the OSS CNI log + // directory, so drop the OSS mount before adding the enterprise one. + c.VolumeMounts = removeVolumeMount(c.VolumeMounts, "cni-log-dir") + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) + } + + mountWindowsPrometheusTLS(ri, ds) +} + +// windowsEnterpriseEnv is the Enterprise felix configuration added to the +// calico-node-windows containers. +func windowsEnterpriseEnv(ri render.Inputs) []corev1.EnvVar { + tls := windowsData(ri).prometheusServerTLS + env := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", installation.NodeReporterPort(ri.FelixConfiguration))}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + if tls != nil && ri.TrustedBundle != nil { + env = append(env, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: tls.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: tls.VolumeMountKeyFilePath()}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: ri.TrustedBundle.MountPath()}, + ) + } + + // Providers without a kube-dns service need a non-default trusted DNS server. + switch ri.Installation.KubernetesProvider { + case operatorv1.ProviderOpenShift: + env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) + case operatorv1.ProviderRKE2: + env = append(env, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}) + } + + return env +} + +// mountWindowsPrometheusTLS mounts the node prometheus reporter keypair onto the +// windows daemonset: the volume, the volume mount on each node container, and +// the pod hash annotation that rolls the pods on cert rotation. +func mountWindowsPrometheusTLS(ri render.Inputs, ds *appsv1.DaemonSet) { + tls := windowsData(ri).prometheusServerTLS + if tls == nil { + return + } + spec := &ds.Spec.Template.Spec + + spec.Volumes = append(spec.Volumes, tls.Volume()) + + for _, c := range windowsNodeContainers(spec) { + c.VolumeMounts = append(c.VolumeMounts, tls.VolumeMount(rmeta.OSTypeWindows)) + } + + if ds.Spec.Template.Annotations == nil { + ds.Spec.Template.Annotations = map[string]string{} + } + ds.Spec.Template.Annotations[tls.HashAnnotationKey()] = tls.HashAnnotationValue() +} + +// windowsNodeMetricsService builds the enterprise-only calico-node-metrics-windows +// Service. +func windowsNodeMetricsService(ri render.Inputs) *corev1.Service { + reporterPort := installation.NodeReporterPort(ri.FelixConfiguration) + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.WindowsNodeMetricsService, + Namespace: common.CalicoNamespace, + Labels: map[string]string{"k8s-app": render.WindowsNodeObjectName}, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"k8s-app": render.WindowsNodeObjectName}, + ClusterIP: "None", + Ports: []corev1.ServicePort{ + { + Name: "calico-metrics-port", + Port: int32(reporterPort), + TargetPort: intstr.FromInt(reporterPort), + Protocol: corev1.ProtocolTCP, + }, + { + Name: "calico-bgp-metrics-port", + Port: render.NodeBGPReporterPort, + TargetPort: intstr.FromInt(int(render.NodeBGPReporterPort)), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } +} + +func removeVolumeMount(mounts []corev1.VolumeMount, name string) []corev1.VolumeMount { + out := mounts[:0] + for _, m := range mounts { + if m.Name != name { + out = append(out, m) + } + } + return out +} diff --git a/pkg/enterprise/windows/extension_test.go b/pkg/enterprise/windows/extension_test.go new file mode 100644 index 0000000000..4055727d6e --- /dev/null +++ b/pkg/enterprise/windows/extension_test.go @@ -0,0 +1,170 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" +) + +var _ = Describe("windows enterprise image override", func() { + ent := &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise} + calico := &operatorv1.InstallationSpec{Variant: operatorv1.Calico} + + It("selects the enterprise windows images for the enterprise variant", func() { + Expect(ext.Windows().Images().Resolve(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, ent)).To(Equal(components.ComponentTigeraNodeWindows)) + Expect(ext.Windows().Images().Resolve(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, ent)).To(Equal(components.ComponentTigeraCNIWindows)) + }) + + It("leaves the defaults in place for the Calico variant", func() { + Expect(ext.Windows().Images().Resolve(render.ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, calico)).To(Equal(components.ComponentCalicoNodeWindows)) + Expect(ext.Windows().Images().Resolve(render.ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, calico)).To(Equal(components.ComponentCalicoCNIWindows)) + }) +}) + +var _ = Describe("windows enterprise modifier", func() { + // newObjs returns a windows daemonset with the node containers and the OSS + // cni-log-dir mount the modifier swaps out. + newObjs := func() []client.Object { + nodeContainer := func(name string) corev1.Container { + return corev1.Container{ + Name: name, + VolumeMounts: []corev1.VolumeMount{{MountPath: "/var/log/calico/cni", Name: "cni-log-dir"}}, + } + } + return []client.Object{ + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: common.WindowsDaemonSetName}, + Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{nodeContainer("felix"), nodeContainer("node"), nodeContainer("confd")}, + }}}, + }, + } + } + + ds := func(objs []client.Object) *appsv1.DaemonSet { + d, _ := extensions.FindObject[*appsv1.DaemonSet](objs, common.WindowsDaemonSetName) + return d + } + container := func(d *appsv1.DaemonSet, name string) *corev1.Container { + for i := range d.Spec.Template.Spec.Containers { + if d.Spec.Template.Spec.Containers[i].Name == name { + return &d.Spec.Template.Spec.Containers[i] + } + } + return nil + } + + ctxFor := func(provider operatorv1.Provider) render.Inputs { + return render.Inputs{ + Installation: &operatorv1.InstallationSpec{Variant: operatorv1.CalicoEnterprise, KubernetesProvider: provider}, + } + } + + It("appends the node-metrics service", func() { + out, _ := ext.Windows().Modify(extensionstest.WindowsStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, ctxFor(operatorv1.ProviderNone)).Objects() + svc, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) + Expect(ok).To(BeTrue()) + Expect(svc.Namespace).To(Equal(common.CalicoNamespace)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(9081))) + }) + + It("swaps the cni log mount for the calico log volume and adds enterprise env", func() { + out, _ := ext.Windows().Modify(extensionstest.WindowsStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, ctxFor(operatorv1.ProviderNone)).Objects() + d := ds(out) + + Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(HaveField("Name", "var-log-calico"))) + for _, name := range []string{"felix", "node", "confd"} { + c := container(d, name) + Expect(c.VolumeMounts).To(ContainElement(HaveField("Name", "var-log-calico"))) + Expect(c.VolumeMounts).NotTo(ContainElement(HaveField("Name", "cni-log-dir"))) + Expect(c.Env).To(ContainElements( + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + corev1.EnvVar{Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + )) + } + }) + + It("sets the trusted DNS server on openshift", func() { + out, _ := ext.Windows().Modify(extensionstest.WindowsStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, ctxFor(operatorv1.ProviderOpenShift)).Objects() + Expect(container(ds(out), "node").Env).To(ContainElement(corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"})) + }) + + It("mounts the prometheus reporter keypair when present", func() { + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + cm, err := certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + tls, err := cm.GetOrCreateKeyPair(cli, render.NodePrometheusTLSServerSecret, common.OperatorNamespace(), []string{"calico-node-metrics-windows"}) + Expect(err).NotTo(HaveOccurred()) + // The installation controller persists the secret; do the same here so the + // windows extension's GetKeyPair finds it. + Expect(cli.Create(context.Background(), tls.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + bundle := cm.CreateTrustedBundle() + + // Build the render inputs the way the windows controller does: run the + // windows extension, which fetches the keypair into the inputs. + ci := controller.Inputs{ + RenderInputs: render.Inputs{ + Installation: ctxFor(operatorv1.ProviderNone).Installation, + TrustedBundle: bundle, + ClusterDomain: dns.DefaultClusterDomain, + }, + Client: cli, + CertificateManager: cm, + } + eci, _, err := ext.Windows().ExtendInputs(ctx, ci) + ri := eci.RenderInputs + Expect(err).NotTo(HaveOccurred()) + + out, _ := ext.Windows().Modify(extensionstest.WindowsStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, ri).Objects() + d := ds(out) + + Expect(d.Spec.Template.Spec.Volumes).To(ContainElement(tls.Volume())) + Expect(d.Spec.Template.Annotations).To(HaveKey(tls.HashAnnotationKey())) + Expect(container(d, "node").Env).To(ContainElement(HaveField("Name", "FELIX_PROMETHEUSREPORTERCERTFILE"))) + Expect(container(d, "node").VolumeMounts).To(ContainElement(tls.VolumeMount(render.Windows(&render.WindowsConfiguration{}).SupportedOSType()))) + }) + + It("does nothing when the operator runs as Calico", func() { + ctx := render.Inputs{Installation: &operatorv1.InstallationSpec{Variant: operatorv1.Calico}} + out, _ := calicoExt.Windows().Modify(extensionstest.WindowsStub{StubComponent: extensionstest.StubComponent{Create: newObjs(), Delete: nil}, Cfg: nil}, ctx).Objects() + _, ok := extensions.FindObject[*corev1.Service](out, render.WindowsNodeMetricsService) + Expect(ok).To(BeFalse()) + Expect(ds(out).Spec.Template.Spec.Volumes).To(BeEmpty()) + }) +}) diff --git a/pkg/enterprise/windows/suite_test.go b/pkg/enterprise/windows/suite_test.go new file mode 100644 index 0000000000..5588875c11 --- /dev/null +++ b/pkg/enterprise/windows/suite_test.go @@ -0,0 +1,41 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/enterprise" + eoptions "github.com/tigera/operator/pkg/enterprise/options" +) + +// calicoExt is the same Enterprise build running as Calico. +var ( + ext = enterprise.New(operatorv1.CalicoEnterprise, eoptions.Options{}) + calicoExt = enterprise.New(operatorv1.Calico, eoptions.Options{}) +) + +// ctx is the reconcile context the specs pass to the extension hooks. +var ctx = context.Background() + +func TestWindows(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/enterprise/windows Suite") +} diff --git a/pkg/enterprise/windows/windows_render_test.go b/pkg/enterprise/windows/windows_render_test.go new file mode 100644 index 0000000000..c2c67d8f8c --- /dev/null +++ b/pkg/enterprise/windows/windows_render_test.go @@ -0,0 +1,937 @@ +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package windows_test + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +var ( + bgpEnabled = operatorv1.BGPEnabled + bgpDisabled = operatorv1.BGPDisabled + logSeverity = operatorv1.LogLevelDebug + logFileMaxAgeDays = uint32(5) + logFileMaxCount = uint32(5) + logFileMaxSize = resource.MustParse("1Mi") +) + +// renderWindows renders the windows component and applies the registered +// enterprise modifier the way the componentHandler does, so enterprise tests +// exercise the integrated output (image overrides come from ResolveImages; the +// metrics service, env, volumes and mounts come from the modifier). +func renderWindows(cfg *render.WindowsConfiguration) []client.Object { + comp := render.Windows(cfg) + ExpectWithOffset(1, comp.ResolveImages(nil)).To(BeNil()) + objs, _ := comp.Objects() + ri := render.Inputs{Installation: cfg.Installation} + out, _ := ext.Windows().Modify(extensionstest.WindowsStub{StubComponent: extensionstest.StubComponent{Create: objs, Delete: nil}, Cfg: nil}, ri).Objects() + return out +} + +func getTyphaNodeTLS(cli client.Client, certificateManager certificatemanager.CertificateManager) *render.TyphaNodeTLS { + nodeKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.NodeTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName, common.OperatorNamespace(), []string{render.FelixCommonName}) + Expect(err).NotTo(HaveOccurred()) + + typhaNonClusterHostKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TyphaTLSSecretName+render.TyphaNonClusterHostSuffix, common.OperatorNamespace(), []string{render.FelixCommonName + render.TyphaNonClusterHostSuffix}) + Expect(err).NotTo(HaveOccurred()) + + trustedBundle := certificateManager.CreateTrustedBundle(nodeKeyPair, typhaKeyPair) + + return &render.TyphaNodeTLS{ + TrustedBundle: trustedBundle, + TyphaSecret: typhaKeyPair, + TyphaSecretNonClusterHost: typhaNonClusterHostKeyPair, + TyphaCommonName: render.TyphaCommonName, + NodeSecret: nodeKeyPair, + NodeCommonName: render.FelixCommonName, + } +} + +// verifyWindowsProbesAndLifecycle asserts the expected node liveness and readiness probe plus pod lifecycle settings. +// The unused argument is kept temporarily so existing call sites compile while the OSS/Enterprise distinction +// is being phased out. +func verifyWindowsProbesAndLifecycle(ds *appsv1.DaemonSet, _ bool) { + livenessCmd := []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/calico.exe", "component", "node", "health", "--felix-live"} + readinessCmd := []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/calico.exe", "component", "node", "health", "--felix-ready"} + preStopCmd := []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/calico.exe", "component", "node", "shutdown"} + + expectedLiveness := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{Command: livenessCmd}, + }, + InitialDelaySeconds: 10, + FailureThreshold: 6, + TimeoutSeconds: 10, + PeriodSeconds: 10, + } + ExpectWithOffset(1, rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").LivenessProbe).To(Equal(expectedLiveness)) + + expectedReadiness := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{Command: readinessCmd}, + }, + TimeoutSeconds: 10, + PeriodSeconds: 10, + } + ExpectWithOffset(1, rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").ReadinessProbe).To(Equal(expectedReadiness)) + + expectedLifecycle := &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{Command: preStopCmd}, + }, + } + ExpectWithOffset(1, rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Lifecycle).To(Equal(expectedLifecycle)) +} + +var _ = Describe("Windows enterprise rendering tests", func() { + var defaultInstance *operatorv1.InstallationSpec + var typhaNodeTLS *render.TyphaNodeTLS + var k8sServiceEp k8sapi.ServiceEndpoint + one := intstr.FromInt(1) + defaultNumExpectedResources := 2 + const defaultClusterDomain = "svc.cluster.local" + var defaultMode int32 = 420 + var cfg render.WindowsConfiguration + var cli client.Client + + BeforeEach(func() { + ff := true + hp := operatorv1.HostPortsEnabled + miMode := operatorv1.MultiInterfaceModeNone + defaultInstance = &operatorv1.InstallationSpec{ + CNI: &operatorv1.CNISpec{ + Type: "Calico", + IPAM: &operatorv1.IPAMSpec{Type: "Calico"}, + }, + CalicoNetwork: &operatorv1.CalicoNetworkSpec{ + BGP: &bgpEnabled, + IPPools: []operatorv1.IPPool{}, + NodeAddressAutodetectionV4: &operatorv1.NodeAddressAutodetection{}, + NodeAddressAutodetectionV6: &operatorv1.NodeAddressAutodetection{}, + HostPorts: &hp, + MultiInterfaceMode: &miMode, + }, + NodeUpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: &one, + }, + }, + Logging: &operatorv1.Logging{ + CNI: &operatorv1.CNILogging{ + LogSeverity: &logSeverity, + LogFileMaxSize: &logFileMaxSize, + LogFileMaxAgeDays: &logFileMaxAgeDays, + LogFileMaxCount: &logFileMaxCount, + }, + }, + WindowsNodes: &operatorv1.WindowsNodeSpec{ + CNIBinDir: "/opt/cni/bin", + CNIConfigDir: "/etc/cni/net.d", + CNILogDir: "/var/log/calico/cni", + }, + } + defaultInstance.CalicoNetwork.IPPools = append(defaultInstance.CalicoNetwork.IPPools, operatorv1.IPPool{CIDR: "192.168.1.0/16", Encapsulation: operatorv1.EncapsulationVXLAN}) + defaultInstance.CalicoNetwork.NodeAddressAutodetectionV4 = &operatorv1.NodeAddressAutodetection{FirstFound: &ff} + defaultInstance.ServiceCIDRs = []string{"10.96.0.0/12"} + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + certificateManager, err := certificatemanager.Create(cli, nil, defaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + // Create a dummy secret to pass as input. + typhaNodeTLS = getTyphaNodeTLS(cli, certificateManager) + + // Dummy service endpoint for k8s API. + k8sServiceEp = k8sapi.ServiceEndpoint{ + Host: "1.2.3.4", + Port: "6443", + } + + // Create a default configuration. + cfg = render.WindowsConfiguration{ + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{"10.96.0.10"}, + Installation: defaultInstance, + ClusterDomain: defaultClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, + ImageOverrides: ext.Windows().Images(), + } + }) + + It("should render all resources for a default configuration using CalicoEnterprise", func() { + type testConf struct { + EnableBGP bool + EnableVXLAN bool + } + for _, testConfig := range []testConf{ + {true, false}, + {false, true}, + {true, true}, + } { + enableBGP := testConfig.EnableBGP + enableVXLAN := testConfig.EnableVXLAN + + if enableBGP { + defaultInstance.CalicoNetwork.BGP = &bgpEnabled + } else { + defaultInstance.CalicoNetwork.BGP = &bgpDisabled + } + + if enableVXLAN { + defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationVXLAN + } else { + defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationNone + } + By(fmt.Sprintf("BGP enabled: %v, VXLAN enabled: %v", enableBGP, enableVXLAN), func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, + {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, + } + defaultInstance.Variant = operatorv1.CalicoEnterprise + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The DaemonSet should have the correct configuration. + ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + // The calico-node-windows daemonset has 3 containers (felix, node and confd). + // confd is only instantiated if using BGP. + numContainers := 3 + if !enableBGP { + numContainers = 2 + } + Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(numContainers)) + for _, container := range ds.Spec.Template.Spec.Containers { + + // Windows node image override results in correct image. + Expect(container.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(container.SecurityContext.Capabilities).To(BeNil()) + Expect(container.SecurityContext.Privileged).To(BeNil()) + Expect(container.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(container.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*container.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*container.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(container.SecurityContext.RunAsUser).To(BeNil()) + Expect(container.SecurityContext.RunAsGroup).To(BeNil()) + Expect(container.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(container.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(container.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(container.SecurityContext.ProcMount).To(BeNil()) + Expect(container.SecurityContext.SeccompProfile).To(BeNil()) + } + + felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") + + // Windows node image override results in correct image. + Expect(felixContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(felixContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(felixContainer.SecurityContext.Privileged).To(BeNil()) + Expect(felixContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(felixContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*felixContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*felixContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(felixContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(felixContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(felixContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(felixContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(felixContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(felixContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(felixContainer.SecurityContext.SeccompProfile).To(BeNil()) + + nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") + + // Windows node image override results in correct image. + Expect(nodeContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(nodeContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(nodeContainer.SecurityContext.Privileged).To(BeNil()) + Expect(nodeContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(nodeContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*nodeContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*nodeContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(nodeContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(nodeContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(nodeContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(nodeContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(nodeContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(nodeContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(nodeContainer.SecurityContext.SeccompProfile).To(BeNil()) + + if enableBGP { + confdContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd") + + // Windows node image override results in correct image. + Expect(confdContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + Expect(confdContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(confdContainer.SecurityContext.Privileged).To(BeNil()) + Expect(confdContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(confdContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*confdContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*confdContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(confdContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(confdContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(confdContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(confdContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(confdContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(confdContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(confdContainer.SecurityContext.SeccompProfile).To(BeNil()) + } + + // Validate correct number of init containers. + Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + + // CNI container uses image override. + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") + Expect(cniContainer.Image).To(Equal(components.TigeraRegistry + "tigera/cni-windows:" + components.ComponentTigeraCNIWindows.Version)) + + Expect(cniContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(cniContainer.SecurityContext.Privileged).To(BeNil()) + Expect(cniContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(cniContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*cniContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*cniContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(cniContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(cniContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(cniContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(cniContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(cniContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(cniContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(cniContainer.SecurityContext.SeccompProfile).To(BeNil()) + + // uninstall container uses image override. + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) + + Expect(uninstallContainer.SecurityContext.Capabilities).To(BeNil()) + Expect(uninstallContainer.SecurityContext.Privileged).To(BeNil()) + Expect(uninstallContainer.SecurityContext.SELinuxOptions).To(BeNil()) + Expect(uninstallContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) + Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) + Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) + Expect(*uninstallContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) + Expect(*uninstallContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) + Expect(uninstallContainer.SecurityContext.RunAsUser).To(BeNil()) + Expect(uninstallContainer.SecurityContext.RunAsGroup).To(BeNil()) + Expect(uninstallContainer.SecurityContext.RunAsNonRoot).To(BeNil()) + Expect(uninstallContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) + Expect(uninstallContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) + Expect(uninstallContainer.SecurityContext.ProcMount).To(BeNil()) + Expect(uninstallContainer.SecurityContext.SeccompProfile).To(BeNil()) + + // Verify env + expectedNodeEnv := []corev1.EnvVar{ + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "WAIT_FOR_DATASTORE", Value: "true"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_HEALTHENABLED", Value: "true"}, + { + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, + {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, + {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, + {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, + {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, + {Name: "FELIX_TYPHACN", Value: "typha-server"}, + {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + + {Name: "VXLAN_VNI", Value: "4096"}, + {Name: "VXLAN_ADAPTER", Value: ""}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + + // Tigera-specific envvars + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + } + + // Set CALICO_NETWORKING_BACKEND + if enableBGP { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}) + } else if enableVXLAN { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}) + } else { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "none"}) + } + + // Set CLUSTER_TYPE + if enableBGP { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}) + } else { + expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,windows"}) + } + + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) + if enableBGP { + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) + } + + // Expect the SECURITY_GROUP env variables to not be set + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + if enableBGP { + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + } + + expectedCNIEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, + {Name: "VXLAN_VNI", Value: "4096"}, + { + Name: "KUBERNETES_NODE_NAME", + Value: "", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "CNI_NETWORK_CONFIG", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + Key: "config", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cni-config-windows", + }, + }, + }, + }, + + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, + {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) + + expectedUninstallEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) + + // Verify volumes. + fileOrCreate := corev1.HostPathFileOrCreate + dirOrCreate := corev1.HostPathDirectoryOrCreate + expectedVols := []corev1.Volume{ + {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, + {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, + { + Name: "tigera-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ca-bundle", + }, + }, + }, + }, + { + Name: render.NodeTLSSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: render.NodeTLSSecretName, + DefaultMode: &defaultMode, + }, + }, + }, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, + } + Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + + // Verify volume mounts. + expectedNodeVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + {MountPath: "/var/run/calico", Name: "var-run-calico"}, + {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, + {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, + {MountPath: "/var/log/calico", Name: "var-log-calico", ReadOnly: false}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + if enableBGP { + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + } + + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + + // Verify tolerations. + Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) + + // Verify readiness and liveness probes. + verifyWindowsProbesAndLifecycle(ds, false) + }) + } + }) + + It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, + {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, + } + + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(len(expectedResources))) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The DaemonSet should have the correct configuration. + ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") + Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") + Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + + // FIXME: confirm openshift CNI path defaults + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + // FIXME: confirm openshift CNI path defaults + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + + // Verify volumes + // FIXME: confirm openshift CNI path defaults + fileOrCreate := corev1.HostPathFileOrCreate + dirOrCreate := corev1.HostPathDirectoryOrCreate + expectedVols := []corev1.Volume{ + {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, + {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, + { + Name: "tigera-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ca-bundle", + }, + }, + }, + }, + { + Name: render.NodeTLSSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: render.NodeTLSSecretName, + DefaultMode: &defaultMode, + }, + }, + }, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, + } + Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + + expectedNodeEnv := []corev1.EnvVar{ + // Default envvars. + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "WAIT_FOR_DATASTORE", Value: "true"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, + {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_HEALTHENABLED", Value: "true"}, + { + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, + {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, + {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, + {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, + {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, + {Name: "FELIX_TYPHACN", Value: "typha-server"}, + {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + + {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}, + + // Tigera-specific envvars + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + + // Calico Windows specific envvars + {Name: "VXLAN_VNI", Value: "4096"}, + {Name: "VXLAN_ADAPTER", Value: ""}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) + + verifyWindowsProbesAndLifecycle(ds, false) + }) + + It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { + expectedResources := []struct { + name string + ns string + group string + version string + kind string + }{ + {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, + {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, + } + + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) + + // Should render the correct resources. + i := 0 + for _, expectedRes := range expectedResources { + rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) + i++ + } + + // The DaemonSet should have the correct configuration. + ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") + Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") + Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + + // FIXME: confirm RKE2 CNI path defaults + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + // FIXME: confirm RKE2 CNI path defaults + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + + // Verify volumes + // FIXME: confirm RKE2 CNI path defaults + fileOrCreate := corev1.HostPathFileOrCreate + dirOrCreate := corev1.HostPathDirectoryOrCreate + expectedVols := []corev1.Volume{ + {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, + {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, + { + Name: "tigera-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "tigera-ca-bundle", + }, + }, + }, + }, + { + Name: render.NodeTLSSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: render.NodeTLSSecretName, + DefaultMode: &defaultMode, + }, + }, + }, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, + } + Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + + expectedNodeEnv := []corev1.EnvVar{ + // Default envvars. + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "DATASTORE_TYPE", Value: "kubernetes"}, + {Name: "WAIT_FOR_DATASTORE", Value: "true"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}, + {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_HEALTHENABLED", Value: "true"}, + { + Name: "NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, + {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, + {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, + {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, + {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, + {Name: "FELIX_TYPHACN", Value: "typha-server"}, + {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + + {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}, + + // Tigera-specific envvars + {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, + {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, + {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, + {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, + {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, + {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, + {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + + // Calico Windows specific envvars + {Name: "VXLAN_VNI", Value: "4096"}, + {Name: "VXLAN_ADAPTER", Value: ""}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + } + Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) + Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) + + verifyWindowsProbesAndLifecycle(ds, false) + + // The metrics service should have the correct configuration. + ms := rtest.GetResource(resources, "calico-node-metrics-windows", "calico-system", "", "v1", "Service").(*corev1.Service) + Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") + }) + + It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.NodeMetricsPort = nil + + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) + + notExpectedEnvVar := corev1.EnvVar{Name: "FELIX_PROMETHEUSMETRICSPORT"} + ds := dsResource.(*appsv1.DaemonSet) + Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) + + // It should have the reporter port, though. + expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} + Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) + }) + + It("should set FELIX_PROMETHEUSMETRICSPORT with a custom value if NodeMetricsPort is set", func() { + var nodeMetricsPort int32 = 1234 + defaultInstance.Variant = operatorv1.CalicoEnterprise + defaultInstance.NodeMetricsPort = &nodeMetricsPort + resources := renderWindows(&cfg) + Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) + + // Assert on expected env vars. + expectedEnvVars := []corev1.EnvVar{ + {Name: "FELIX_PROMETHEUSMETRICSPORT", Value: "1234"}, + {Name: "FELIX_PROMETHEUSMETRICSENABLED", Value: "true"}, + } + ds := dsResource.(*appsv1.DaemonSet) + for _, v := range expectedEnvVars { + Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(v)) + } + + // Assert we set annotations properly. + Expect(ds.Spec.Template.Annotations["prometheus.io/scrape"]).To(Equal("true")) + Expect(ds.Spec.Template.Annotations["prometheus.io/port"]).To(Equal("1234")) + }) +}) diff --git a/pkg/extensions/apiserver.go b/pkg/extensions/apiserver.go new file mode 100644 index 0000000000..cf3b10ec66 --- /dev/null +++ b/pkg/extensions/apiserver.go @@ -0,0 +1,48 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "context" + + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// APIServerExtension is the variant's hook into the apiserver controller. +type APIServerExtension interface { + ExtendInputs(ctx context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) + Watches(c ctrlruntime.Controller) error + + // Modify layers the variant onto a component the controller rendered. + Modify(c render.Component, ri render.Inputs) render.Component +} + +// noopAPIServer runs the core operator's behavior unchanged. +type noopAPIServer struct{} + +func (noopAPIServer) ExtendInputs(_ context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) { + return ci, nil, nil +} + +func (noopAPIServer) Watches(ctrlruntime.Controller) error { + return nil +} + +func (noopAPIServer) Modify(c render.Component, _ render.Inputs) render.Component { + return c +} diff --git a/pkg/extensions/clusterconnection.go b/pkg/extensions/clusterconnection.go new file mode 100644 index 0000000000..4a32534cf3 --- /dev/null +++ b/pkg/extensions/clusterconnection.go @@ -0,0 +1,69 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "context" + + "k8s.io/client-go/kubernetes" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// ClusterConnectionExtension is the variant's hook into the clusterconnection +// controller and the guardian components it renders. +type ClusterConnectionExtension interface { + ExtendInputs(ctx context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) + + // ValidateAndDefault rejects fields the variant does not support and fills in + // the ones it defaults. + ValidateAndDefault(cr *operatorv1.ManagementClusterConnection) error + // Watches registers the variant's watches. The clientset is for those that must + // wait on a CRD. + Watches(c ctrlruntime.Controller, cs kubernetes.Interface) error + + // Modify layers the variant onto a component the controller rendered. + Modify(c render.Component, ri render.Inputs) render.Component +} + +// noopClusterConnection runs the core operator's behavior unchanged. +type noopClusterConnection struct{} + +func (noopClusterConnection) ExtendInputs(_ context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) { + return ci, nil, nil +} + +func (noopClusterConnection) Watches(ctrlruntime.Controller, kubernetes.Interface) error { + return nil +} + +// ValidateAndDefault rejects the Enterprise-only fields of the shared CRD. +func (noopClusterConnection) ValidateAndDefault(cr *operatorv1.ManagementClusterConnection) error { + if cr.Spec.Impersonation != nil { + return InvalidConfigf("ManagementClusterConnection.Spec.Impersonation must be unset when Installation.Spec.Variant = Calico") + } + if cr.Spec.TLS != nil && cr.Spec.TLS.CA == operatorv1.CATypePublic { + return InvalidConfigf("Guardian CA cannot be public in Calico") + } + return nil +} + +func (noopClusterConnection) Modify(c render.Component, _ render.Inputs) render.Component { + return c +} diff --git a/pkg/extensions/component.go b/pkg/extensions/component.go new file mode 100644 index 0000000000..803d8fdc03 --- /dev/null +++ b/pkg/extensions/component.go @@ -0,0 +1,57 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/render" +) + +// Modifier post-processes the objects a render component produced. Appending to +// delete cleans up what a prior variant left behind. +type Modifier func(create, delete []client.Object) (newCreate, newDelete []client.Object) + +// Decorate wraps base so modify runs over its rendered objects. An Installation +// asking for a different variant gets base untouched. +func Decorate(base render.Component, ri render.Inputs, variant operatorv1.ProductVariant, modify Modifier) render.Component { + if ri.Installation == nil || ri.Installation.Variant != variant { + return base + } + return &decoratedComponent{Component: base, modify: modify} +} + +// decoratedComponent renders its base component and runs the modifier over the result. +type decoratedComponent struct { + render.Component + + modify Modifier +} + +func (d *decoratedComponent) Objects() ([]client.Object, []client.Object) { + return d.modify(d.Component.Objects()) +} + +// FindObject returns the first object of type T with the given name. +func FindObject[T client.Object](objs []client.Object, name string) (T, bool) { + var zero T + for _, o := range objs { + if t, ok := o.(T); ok && o.GetName() == name { + return t, true + } + } + return zero, false +} diff --git a/pkg/extensions/doc.go b/pkg/extensions/doc.go new file mode 100644 index 0000000000..cfbfc01586 --- /dev/null +++ b/pkg/extensions/doc.go @@ -0,0 +1,31 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extensions is the seam other product variants (today just Calico +// Enterprise) use to layer variant-specific behavior onto the core operator's +// render output, so core code never branches on variant. +// +// Extensions holds one extension per controller a variant extends. Each is an +// interface the variant implements, covering two phases. +// +// ExtendInputs is the controller phase. It has cluster access via controller.Inputs +// and does the side-effecting work a render hook can't: rejecting unsupported +// config, creating certificates, extending the trusted bundle. +// +// The Modify methods are the render phase: pure hooks handed the component and the +// same typed config the core operator rendered it from, returning a component whose +// objects the variant has adjusted. Decorate does the wrapping. +// +// A variant builds the whole set in one place at startup - see pkg/enterprise. +package extensions diff --git a/pkg/extensions/errors.go b/pkg/extensions/errors.go new file mode 100644 index 0000000000..c612b58371 --- /dev/null +++ b/pkg/extensions/errors.go @@ -0,0 +1,58 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "errors" + "fmt" + + operatorv1 "github.com/tigera/operator/api/v1" +) + +// degraded carries the status reason a controller reports for an error the +// extension returned. +type degraded struct { + error + reason operatorv1.TigeraStatusReason +} + +func (e degraded) Unwrap() error { + return e.error +} + +// Degradedf returns an error the controller degrades with under the given reason. +func Degradedf(reason operatorv1.TigeraStatusReason, format string, args ...any) error { + return degraded{error: fmt.Errorf(format, args...), reason: reason} +} + +// InvalidConfigf reports configuration the variant does not support. +func InvalidConfigf(format string, args ...any) error { + return Degradedf(operatorv1.ResourceValidationError, format, args...) +} + +// NotReadyf reports a dependency the extension is waiting on. Controllers wait for +// a watch rather than failing. +func NotReadyf(format string, args ...any) error { + return Degradedf(operatorv1.ResourceNotReady, format, args...) +} + +// DegradedReason returns the reason an extension attached to err, if it attached one. +func DegradedReason(err error) (operatorv1.TigeraStatusReason, bool) { + var d degraded + if errors.As(err, &d) { + return d.reason, true + } + return "", false +} diff --git a/pkg/extensions/extensions.go b/pkg/extensions/extensions.go new file mode 100644 index 0000000000..506482f87e --- /dev/null +++ b/pkg/extensions/extensions.go @@ -0,0 +1,64 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +// Set is the extensions a variant supplies, one per extended controller. What it +// leaves unset runs the core behavior. +type Set struct { + Installation InstallationExtension + Windows WindowsExtension + APIServer APIServerExtension + ClusterConnection ClusterConnectionExtension +} + +// Extensions is the variant behavior the operator runs with. The zero value extends +// nothing, so the accessors never return nil. +type Extensions struct { + set Set +} + +// New returns the extensions the operator runs with. A variant builds this once at +// startup; see pkg/enterprise. +func New(s Set) Extensions { + return Extensions{set: s} +} + +func (e Extensions) Installation() InstallationExtension { + if e.set.Installation == nil { + return noopInstallation{} + } + return e.set.Installation +} + +func (e Extensions) Windows() WindowsExtension { + if e.set.Windows == nil { + return noopWindows{} + } + return e.set.Windows +} + +func (e Extensions) APIServer() APIServerExtension { + if e.set.APIServer == nil { + return noopAPIServer{} + } + return e.set.APIServer +} + +func (e Extensions) ClusterConnection() ClusterConnectionExtension { + if e.set.ClusterConnection == nil { + return noopClusterConnection{} + } + return e.set.ClusterConnection +} diff --git a/pkg/extensions/extensions_suite_test.go b/pkg/extensions/extensions_suite_test.go new file mode 100644 index 0000000000..791eedb737 --- /dev/null +++ b/pkg/extensions/extensions_suite_test.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestExtensions(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "pkg/extensions Suite") +} diff --git a/pkg/extensions/extensions_test.go b/pkg/extensions/extensions_test.go new file mode 100644 index 0000000000..67e8890570 --- /dev/null +++ b/pkg/extensions/extensions_test.go @@ -0,0 +1,126 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/extensions" + "github.com/tigera/operator/pkg/extensions/extensionstest" + "github.com/tigera/operator/pkg/render" +) + +func configMap(name string) client.Object { + return &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name}} +} + +func baseComponent() render.Component { + return extensionstest.StubComponent{Create: []client.Object{configMap("base")}} +} + +func inputsFor(variant operatorv1.ProductVariant) render.Inputs { + return render.Inputs{Installation: &operatorv1.InstallationSpec{Variant: variant}} +} + +func addConfigMap(create, del []client.Object) ([]client.Object, []client.Object) { + return append(create, configMap("added")), del +} + +var _ = Describe("Decorate", func() { + It("runs the modifier over what the component rendered", func() { + c := extensions.Decorate(baseComponent(), inputsFor(operatorv1.CalicoEnterprise), operatorv1.CalicoEnterprise, addConfigMap) + + create, _ := c.Objects() + Expect(create).To(HaveLen(2)) + _, ok := extensions.FindObject[*corev1.ConfigMap](create, "added") + Expect(ok).To(BeTrue()) + }) + + It("leaves the component alone when the Installation asks for another variant", func() { + c := extensions.Decorate(baseComponent(), inputsFor(operatorv1.Calico), operatorv1.CalicoEnterprise, addConfigMap) + + create, _ := c.Objects() + Expect(create).To(HaveLen(1)) + }) + + It("leaves the component alone when there is no Installation", func() { + c := extensions.Decorate(baseComponent(), render.Inputs{}, operatorv1.CalicoEnterprise, addConfigMap) + + create, _ := c.Objects() + Expect(create).To(HaveLen(1)) + }) +}) + +var _ = Describe("the zero value Extensions", func() { + It("runs the base behavior for every controller", func() { + var e extensions.Extensions + + Expect(e.Installation().ProductVersion()).NotTo(BeEmpty()) + Expect(e.Installation().Images()).To(BeNil()) + Expect(e.Windows().Watches(nil)).NotTo(HaveOccurred()) + + ci, keyPairs, err := e.ClusterConnection().ExtendInputs(context.Background(), controller.Inputs{}) + Expect(err).NotTo(HaveOccurred()) + Expect(keyPairs).To(BeEmpty()) + Expect(ci.RenderInputs.Extension).To(BeNil()) + + create, _ := e.APIServer().Modify(baseComponent(), render.Inputs{}).Objects() + Expect(create).To(HaveLen(1)) + }) +}) + +var _ = Describe("the base ManagementClusterConnection validation", func() { + var e extensions.Extensions + + It("accepts a connection that uses none of the Enterprise fields", func() { + cr := &operatorv1.ManagementClusterConnection{} + Expect(e.ClusterConnection().ValidateAndDefault(cr)).NotTo(HaveOccurred()) + Expect(cr.Spec.Impersonation).To(BeNil()) + }) + + It("rejects impersonation, which only Enterprise Voltron honors", func() { + cr := &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{Impersonation: &operatorv1.Impersonation{}}, + } + Expect(e.ClusterConnection().ValidateAndDefault(cr)).To(MatchError(ContainSubstring("Impersonation must be unset"))) + }) + + It("rejects a public CA, since only Enterprise guardian trusts the system bundle", func() { + cr := &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + TLS: &operatorv1.ManagementClusterTLS{CA: operatorv1.CATypePublic}, + }, + } + Expect(e.ClusterConnection().ValidateAndDefault(cr)).To(MatchError(ContainSubstring("cannot be public"))) + }) + + It("accepts the Tigera CA", func() { + cr := &operatorv1.ManagementClusterConnection{ + Spec: operatorv1.ManagementClusterConnectionSpec{ + TLS: &operatorv1.ManagementClusterTLS{CA: operatorv1.CATypeTigera}, + }, + } + Expect(e.ClusterConnection().ValidateAndDefault(cr)).NotTo(HaveOccurred()) + }) +}) diff --git a/pkg/extensions/extensionstest/stub.go b/pkg/extensions/extensionstest/stub.go new file mode 100644 index 0000000000..415a0264d4 --- /dev/null +++ b/pkg/extensions/extensionstest/stub.go @@ -0,0 +1,138 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extensionstest holds shared helpers for exercising an extension's Modify +// dispatch against raw object lists. +package extensionstest + +import ( + client "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/kubecontrollers" +) + +// StubComponent adapts raw object lists to a render.Component. The typed stubs embed +// it to satisfy what an extension dispatches on. +type StubComponent struct { + Create, Delete []client.Object +} + +func (s StubComponent) ResolveImages(*operatorv1.ImageSet) error { + return nil +} + +func (s StubComponent) Objects() ([]client.Object, []client.Object) { + return s.Create, s.Delete +} + +func (s StubComponent) Ready() bool { + return true +} + +func (s StubComponent) SupportedOSType() rmeta.OSType { + return rmeta.OSTypeAny +} + +type NodeStub struct { + StubComponent + + Cfg *render.NodeConfiguration +} + +func (s NodeStub) NodeConfig() *render.NodeConfiguration { + return s.Cfg +} + +type TyphaStub struct { + StubComponent + + Cfg *render.TyphaConfiguration +} + +func (s TyphaStub) TyphaConfig() *render.TyphaConfiguration { + return s.Cfg +} + +type WindowsStub struct { + StubComponent + + Cfg *render.WindowsConfiguration +} + +func (s WindowsStub) WindowsConfig() *render.WindowsConfiguration { + return s.Cfg +} + +type GuardianStub struct { + StubComponent + + Cfg *render.GuardianConfiguration +} + +func (s GuardianStub) GuardianConfig() *render.GuardianConfiguration { + return s.Cfg +} + +type GuardianPolicyStub struct { + StubComponent + + Cfg *render.GuardianConfiguration +} + +func (s GuardianPolicyStub) GuardianPolicyConfig() *render.GuardianConfiguration { + return s.Cfg +} + +type APIServerStub struct { + StubComponent + + Cfg *render.APIServerConfiguration +} + +func (s APIServerStub) APIServerConfig() *render.APIServerConfiguration { + return s.Cfg +} + +type APIServerPolicyStub struct { + StubComponent + + Cfg *render.APIServerConfiguration +} + +func (s APIServerPolicyStub) APIServerPolicyConfig() *render.APIServerConfiguration { + return s.Cfg +} + +type KubeControllersStub struct { + StubComponent + + Cfg *kubecontrollers.KubeControllersConfiguration +} + +func (s KubeControllersStub) KubeControllersConfig() *kubecontrollers.KubeControllersConfiguration { + return s.Cfg +} + +type KubeControllersPolicyStub struct { + StubComponent + + Cfg *kubecontrollers.KubeControllersConfiguration +} + +func (s KubeControllersPolicyStub) KubeControllersPolicyConfig() *kubecontrollers.KubeControllersConfiguration { + return s.Cfg +} diff --git a/pkg/extensions/installation.go b/pkg/extensions/installation.go new file mode 100644 index 0000000000..91436a6d15 --- /dev/null +++ b/pkg/extensions/installation.go @@ -0,0 +1,80 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "context" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// InstallationExtension is the variant's hook into the installation controller and +// the components it renders. +type InstallationExtension interface { + // ExtendInputs does the reconcile work render cannot, returning keypairs for the + // controller to manage. Rejects unsupported config with InvalidConfigf. + ExtendInputs(ctx context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) + + // Watches registers the watches the extension needs. + Watches(c ctrlruntime.Controller) error + + // DefaultFelixConfiguration defaults FelixConfiguration fields, reporting whether + // it changed fc. It runs before Felix defaulting persists. + DefaultFelixConfiguration(install *operatorv1.InstallationSpec, fc *v3.FelixConfiguration) (bool, error) + + // ProductVersion is the version the operator writes to the Installation status. + ProductVersion() string + + // Images overrides the images the rendered components resolve to. + Images() *imageoverride.Overrides + + // Modify layers the variant onto a component the controller rendered. + Modify(c render.Component, ri render.Inputs) render.Component +} + +// noopInstallation runs the core operator's behavior unchanged. +type noopInstallation struct{} + +func (noopInstallation) ExtendInputs(_ context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) { + return ci, nil, nil +} + +func (noopInstallation) Watches(ctrlruntime.Controller) error { + return nil +} + +func (noopInstallation) DefaultFelixConfiguration(*operatorv1.InstallationSpec, *v3.FelixConfiguration) (bool, error) { + return false, nil +} + +func (noopInstallation) ProductVersion() string { + return components.CalicoRelease +} + +func (noopInstallation) Images() *imageoverride.Overrides { + return nil +} + +func (noopInstallation) Modify(c render.Component, _ render.Inputs) render.Component { + return c +} diff --git a/pkg/extensions/windows.go b/pkg/extensions/windows.go new file mode 100644 index 0000000000..23f71a032c --- /dev/null +++ b/pkg/extensions/windows.go @@ -0,0 +1,54 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extensions + +import ( + "context" + + "github.com/tigera/operator/pkg/controller" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/imageoverride" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// WindowsExtension is the variant's hook into the windows controller. +type WindowsExtension interface { + ExtendInputs(ctx context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) + Watches(c ctrlruntime.Controller) error + + // Modify layers the variant onto a component the controller rendered. + Modify(c render.Component, ri render.Inputs) render.Component + Images() *imageoverride.Overrides +} + +// noopWindows runs the core operator's behavior unchanged. +type noopWindows struct{} + +func (noopWindows) ExtendInputs(_ context.Context, ci controller.Inputs) (controller.Inputs, []certificatemanagement.KeyPairInterface, error) { + return ci, nil, nil +} + +func (noopWindows) Watches(ctrlruntime.Controller) error { + return nil +} + +func (noopWindows) Images() *imageoverride.Overrides { + return nil +} + +func (noopWindows) Modify(c render.Component, _ render.Inputs) render.Component { + return c +} diff --git a/pkg/imageoverride/imageoverride.go b/pkg/imageoverride/imageoverride.go new file mode 100644 index 0000000000..ee4c48d559 --- /dev/null +++ b/pkg/imageoverride/imageoverride.go @@ -0,0 +1,61 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package imageoverride is a leaf package (no render/operator dependencies) +// that holds the image override table. The render package imports it to resolve +// a component's image without depending on pkg/extensions, which would cycle. +package imageoverride + +import ( + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" +) + +type overrideKey struct { + variant operatorv1.ProductVariant + key string +} + +// Overrides maps a component (keyed by variant) to the image it should resolve +// to, letting a variant swap a component's image without the render package +// branching on variant. The render component holds one and resolves through it. +// Registry, image path, and FIPS handling are applied downstream in the render +// package, so an override only picks which component. +type Overrides struct { + m map[overrideKey]components.Component +} + +// New returns an empty Overrides. +func New() *Overrides { + return &Overrides{m: map[overrideKey]components.Component{}} +} + +// Register stores image under key for the given variant. The key is the render +// component's image identifier (e.g. "node"). +func (o *Overrides) Register(variant operatorv1.ProductVariant, key string, image components.Component) { + o.m[overrideKey{variant, key}] = image +} + +// Resolve returns the override registered for key under the installation's +// variant, otherwise def. It is safe to call on a nil *Overrides (the core +// operator hands render no overrides), which always returns def. +func (o *Overrides) Resolve(key string, def components.Component, in *operatorv1.InstallationSpec) components.Component { + if o == nil || in == nil { + return def + } + if image, ok := o.m[overrideKey{in.Variant, key}]; ok { + return image + } + return def +} diff --git a/pkg/imports/crds/enterprise/01-crd-eck-bundle.yaml b/pkg/imports/crds/enterprise/01-crd-eck-bundle.yaml index 903d3a5156..2bea01734b 100644 --- a/pkg/imports/crds/enterprise/01-crd-eck-bundle.yaml +++ b/pkg/imports/crds/enterprise/01-crd-eck-bundle.yaml @@ -4,12 +4,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: agents.agent.k8s.elastic.co spec: group: agent.k8s.elastic.co @@ -114,6 +114,8 @@ spec: elasticsearchRefs: items: properties: + clientCertificateSecretName: + type: string name: type: string namespace: @@ -130,6 +132,8 @@ spec: type: boolean fleetServerRef: properties: + clientCertificateSecretName: + type: string name: type: string namespace: @@ -263,6 +267,11 @@ spec: secretName: type: string type: object + client: + properties: + authentication: + type: boolean + type: object selfSignedCertificate: properties: disabled: @@ -299,6 +308,39 @@ spec: type: string policyID: type: string + resources: + properties: + limits: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object revisionHistoryLimit: format: int32 type: integer @@ -324,6 +366,9 @@ spec: type: array serviceAccountName: type: string + spaceID: + pattern: ^[a-z0-9_-]+$ + type: string statefulSet: properties: podManagementPolicy: @@ -467,6 +512,23 @@ spec: availableNodes: format: int32 type: integer + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array elasticsearchAssociationsStatus: additionalProperties: type: string @@ -497,12 +559,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: apmservers.apm.k8s.elastic.co spec: group: apm.k8s.elastic.co @@ -550,6 +612,8 @@ spec: type: integer elasticsearchRef: properties: + clientCertificateSecretName: + type: string name: type: string namespace: @@ -715,6 +779,39 @@ spec: podTemplate: type: object x-kubernetes-preserve-unknown-fields: true + resources: + properties: + limits: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object revisionHistoryLimit: format: int32 type: integer @@ -750,6 +847,23 @@ spec: availableNodes: format: int32 type: integer + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array count: format: int32 type: integer @@ -1019,12 +1133,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: autoopsagentpolicies.autoops.k8s.elastic.co spec: group: autoops.k8s.elastic.co @@ -1066,6 +1180,14 @@ spec: secretName: type: string type: object + config: + type: object + x-kubernetes-preserve-unknown-fields: true + configRef: + properties: + secretName: + type: string + type: object image: type: string namespaceSelector: @@ -1123,6 +1245,39 @@ spec: type: object type: object x-kubernetes-map-type: atomic + resources: + properties: + limits: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object revisionHistoryLimit: format: int32 type: integer @@ -1135,6 +1290,23 @@ spec: type: object status: properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array details: additionalProperties: properties: @@ -1179,12 +1351,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: beats.beat.k8s.elastic.co spec: group: beat.k8s.elastic.co @@ -1291,6 +1463,8 @@ spec: type: object elasticsearchRef: properties: + clientCertificateSecretName: + type: string name: type: string namespace: @@ -1348,6 +1522,39 @@ spec: type: array type: object type: object + resources: + properties: + limits: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object revisionHistoryLimit: format: int32 type: integer @@ -1388,6 +1595,23 @@ spec: availableNodes: format: int32 type: integer + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array elasticsearchAssociationStatus: type: string expectedNodes: @@ -1418,12 +1642,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: elasticmapsservers.maps.k8s.elastic.co spec: group: maps.k8s.elastic.co @@ -1476,6 +1700,8 @@ spec: type: integer elasticsearchRef: properties: + clientCertificateSecretName: + type: string name: type: string namespace: @@ -1630,6 +1856,39 @@ spec: podTemplate: type: object x-kubernetes-preserve-unknown-fields: true + resources: + properties: + limits: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object revisionHistoryLimit: format: int32 type: integer @@ -1647,6 +1906,23 @@ spec: availableNodes: format: int32 type: integer + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array count: format: int32 type: integer @@ -1675,12 +1951,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: elasticsearchautoscalers.autoscaling.k8s.elastic.co spec: group: autoscaling.k8s.elastic.co @@ -1932,12 +2208,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: elasticsearches.elasticsearch.k8s.elastic.co spec: group: elasticsearch.k8s.elastic.co @@ -2197,6 +2473,39 @@ spec: podTemplate: type: object x-kubernetes-preserve-unknown-fields: true + resources: + properties: + limits: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object volumeClaimTemplates: items: properties: @@ -3317,12 +3626,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: enterprisesearches.enterprisesearch.k8s.elastic.co spec: group: enterprisesearch.k8s.elastic.co @@ -3375,6 +3684,8 @@ spec: type: integer elasticsearchRef: properties: + clientCertificateSecretName: + type: string name: type: string namespace: @@ -3529,6 +3840,39 @@ spec: podTemplate: type: object x-kubernetes-preserve-unknown-fields: true + resources: + properties: + limits: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object revisionHistoryLimit: format: int32 type: integer @@ -3544,6 +3888,23 @@ spec: availableNodes: format: int32 type: integer + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array count: format: int32 type: integer @@ -3772,6 +4133,23 @@ spec: availableNodes: format: int32 type: integer + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array count: format: int32 type: integer @@ -3795,12 +4173,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: kibanas.kibana.k8s.elastic.co spec: group: kibana.k8s.elastic.co @@ -4059,6 +4437,39 @@ spec: podTemplate: type: object x-kubernetes-preserve-unknown-fields: true + resources: + properties: + limits: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object revisionHistoryLimit: format: int32 type: integer @@ -4096,6 +4507,23 @@ spec: availableNodes: format: int32 type: integer + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array count: format: int32 type: integer @@ -4363,12 +4791,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: logstashes.logstash.k8s.elastic.co spec: group: logstash.k8s.elastic.co @@ -4425,6 +4853,8 @@ spec: elasticsearchRefs: items: properties: + clientCertificateSecretName: + type: string clusterName: minLength: 1 type: string @@ -4490,6 +4920,39 @@ spec: podTemplate: type: object x-kubernetes-preserve-unknown-fields: true + resources: + properties: + limits: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object revisionHistoryLimit: format: int32 type: integer @@ -4868,6 +5331,23 @@ spec: availableNodes: format: int32 type: integer + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array elasticsearchAssociationsStatus: additionalProperties: type: string @@ -4906,12 +5386,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: packageregistries.packageregistry.k8s.elastic.co spec: group: packageregistry.k8s.elastic.co @@ -5107,6 +5587,39 @@ spec: podTemplate: type: object x-kubernetes-preserve-unknown-fields: true + resources: + properties: + limits: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + properties: + cpu: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object revisionHistoryLimit: format: int32 type: integer @@ -5120,6 +5633,23 @@ spec: availableNodes: format: int32 type: integer + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array count: format: int32 type: integer @@ -5148,12 +5678,12 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.21.0 helm.sh/resource-policy: keep labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.1' + app.kubernetes.io/version: '3.5.0' name: stackconfigpolicies.stackconfigpolicy.k8s.elastic.co spec: group: stackconfigpolicy.k8s.elastic.co @@ -5251,6 +5781,9 @@ spec: securityRoleMappings: type: object x-kubernetes-preserve-unknown-fields: true + securityRoles: + type: object + x-kubernetes-preserve-unknown-fields: true snapshotLifecyclePolicies: type: object x-kubernetes-preserve-unknown-fields: true @@ -5331,6 +5864,26 @@ spec: - secretName type: object type: array + variablesFrom: + items: + properties: + kind: + enum: + - ConfigMap + - Secret + type: string + name: + minLength: 1 + type: string + namespace: + type: string + optional: + type: boolean + required: + - kind + - name + type: object + type: array weight: default: 0 format: int32 diff --git a/pkg/imports/crds/enterprise/v1.crd.projectcalico.org/crd.projectcalico.org_networks.yaml b/pkg/imports/crds/enterprise/v1.crd.projectcalico.org/crd.projectcalico.org_networks.yaml index a4807847a5..a2d08f137a 100644 --- a/pkg/imports/crds/enterprise/v1.crd.projectcalico.org/crd.projectcalico.org_networks.yaml +++ b/pkg/imports/crds/enterprise/v1.crd.projectcalico.org/crd.projectcalico.org_networks.yaml @@ -89,6 +89,12 @@ spec: required: - name type: object + x-kubernetes-validations: + - message: + "name must not start with calb- or calq-: + those prefixes are reserved for devices Calico creates + and deletes" + rule: "!self.name.startsWith('calb-') && !self.name.startsWith('calq-')" managedBridge: description: |- ManagedBridge instructs Calico to create and fully manage the diff --git a/pkg/imports/crds/enterprise/v3.projectcalico.org/projectcalico.org_networks.yaml b/pkg/imports/crds/enterprise/v3.projectcalico.org/projectcalico.org_networks.yaml index e460043390..7721bd2488 100644 --- a/pkg/imports/crds/enterprise/v3.projectcalico.org/projectcalico.org_networks.yaml +++ b/pkg/imports/crds/enterprise/v3.projectcalico.org/projectcalico.org_networks.yaml @@ -93,6 +93,12 @@ spec: required: - name type: object + x-kubernetes-validations: + - message: + "name must not start with calb- or calq-: + those prefixes are reserved for devices Calico creates + and deletes" + rule: "!self.name.startsWith('calb-') && !self.name.startsWith('calq-')" managedBridge: description: |- ManagedBridge instructs Calico to create and fully manage the diff --git a/pkg/imports/crds/operator/operator.tigera.io_installations.yaml b/pkg/imports/crds/operator/operator.tigera.io_installations.yaml index d63613ba77..26c4b60ea4 100644 --- a/pkg/imports/crds/operator/operator.tigera.io_installations.yaml +++ b/pkg/imports/crds/operator/operator.tigera.io_installations.yaml @@ -1316,6 +1316,14 @@ spec: type: object type: object type: object + calicoLibHostPath: + default: /var/lib/calico + description: |- + CalicoLibHostPath optionally specifies the host path mounted into calico-node containers at + /var/lib/calico. Pair this with CalicoRunHostPath when Calico data lives under a non-standard + host directory (for example microk8s uses /var/snap/microk8s/current/var/lib/calico). + maxLength: 1024 + type: string calicoNetwork: description: CalicoNetwork specifies networking configuration options @@ -3039,23 +3047,6 @@ spec: type: object type: object type: object - calicoLibHostPath: - default: /var/lib/calico - description: |- - CalicoLibHostPath optionally specifies the host path mounted into calico-node containers at - /var/lib/calico. Pair this with CalicoRunHostPath when Calico data lives under a non-standard - host directory (for example microk8s uses /var/snap/microk8s/current/var/lib/calico). - maxLength: 1024 - type: string - calicoRunHostPath: - default: /var/run/calico - description: |- - CalicoRunHostPath optionally specifies the host path mounted into calico-node containers at - /var/run/calico. Environments such as microk8s place Calico runtime state under a non-standard - host directory (for example /var/snap/microk8s/current/var/run/calico); set this field so the - operator continues using that path after a manifest-to-operator migration. - maxLength: 1024 - type: string calicoNodeWindowsDaemonSet: description: CalicoNodeWindowsDaemonSet configures the calico-node-windows @@ -4402,6 +4393,15 @@ spec: type: object type: object type: object + calicoRunHostPath: + default: /var/run/calico + description: |- + CalicoRunHostPath optionally specifies the host path mounted into calico-node containers at + /var/run/calico. Environments such as microk8s place Calico runtime state under a non-standard + host directory (for example /var/snap/microk8s/current/var/run/calico); set this field so the + operator continues using that path after a manifest-to-operator migration. + maxLength: 1024 + type: string calicoWindowsUpgradeDaemonSet: description: |- Deprecated. The CalicoWindowsUpgradeDaemonSet is deprecated and will be removed from the API in the future. @@ -10692,6 +10692,14 @@ spec: type: object type: object type: object + calicoLibHostPath: + default: /var/lib/calico + description: |- + CalicoLibHostPath optionally specifies the host path mounted into calico-node containers at + /var/lib/calico. Pair this with CalicoRunHostPath when Calico data lives under a non-standard + host directory (for example microk8s uses /var/snap/microk8s/current/var/lib/calico). + maxLength: 1024 + type: string calicoNetwork: description: CalicoNetwork specifies networking configuration @@ -12442,23 +12450,6 @@ spec: type: object type: object type: object - calicoLibHostPath: - default: /var/lib/calico - description: |- - CalicoLibHostPath optionally specifies the host path mounted into calico-node containers at - /var/lib/calico. Pair this with CalicoRunHostPath when Calico data lives under a non-standard - host directory (for example microk8s uses /var/snap/microk8s/current/var/lib/calico). - maxLength: 1024 - type: string - calicoRunHostPath: - default: /var/run/calico - description: |- - CalicoRunHostPath optionally specifies the host path mounted into calico-node containers at - /var/run/calico. Environments such as microk8s place Calico runtime state under a non-standard - host directory (for example /var/snap/microk8s/current/var/run/calico); set this field so the - operator continues using that path after a manifest-to-operator migration. - maxLength: 1024 - type: string calicoNodeWindowsDaemonSet: description: CalicoNodeWindowsDaemonSet configures the calico-node-windows @@ -13828,6 +13819,15 @@ spec: type: object type: object type: object + calicoRunHostPath: + default: /var/run/calico + description: |- + CalicoRunHostPath optionally specifies the host path mounted into calico-node containers at + /var/run/calico. Environments such as microk8s place Calico runtime state under a non-standard + host directory (for example /var/snap/microk8s/current/var/run/calico); set this field so the + operator continues using that path after a manifest-to-operator migration. + maxLength: 1024 + type: string calicoWindowsUpgradeDaemonSet: description: |- Deprecated. The CalicoWindowsUpgradeDaemonSet is deprecated and will be removed from the API in the future. diff --git a/pkg/render/apiserver.go b/pkg/render/apiserver.go index caba0cad5a..36eb872f58 100644 --- a/pkg/render/apiserver.go +++ b/pkg/render/apiserver.go @@ -16,7 +16,6 @@ package render import ( "fmt" - "net/url" "strings" admregv1 "k8s.io/api/admissionregistration/v1" @@ -28,41 +27,33 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" - "github.com/tigera/operator/pkg/render/common/authentication" rcomp "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/podaffinity" - "github.com/tigera/operator/pkg/render/common/rbacmanagement" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) -type ContainerName string - const ( APIServerPort = 5443 APIServerPortName = "apiserver" APIServerPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "apiserver-access" + TieredPolicyPassthruClusterRoleName = "calico-tiered-policy-passthrough" + // APIServiceName is the aggregated APIService that fronts the projectcalico.org/v3 API group. APIServiceName = "v3.projectcalico.org" - - auditLogsVolumeName = "calico-audit-logs" - auditPolicyVolumeName = "calico-audit-policy" ) const ( @@ -74,26 +65,25 @@ const ( QueryserverServiceName = "calico-api" // Use the same API server container name for both OSS and Enterprise. - APIServerName = "calico-apiserver" - APIServerContainerName ContainerName = "calico-apiserver" - TigeraAPIServerQueryServerContainerName ContainerName = "tigera-queryserver" + APIServerName = "calico-apiserver" + APIServerContainerName = "calico-apiserver" + TigeraAPIServerQueryServerContainerName = "tigera-queryserver" CalicoAPIServerTLSSecretName = "calico-apiserver-certs" APIServerServiceName = "calico-api" APIServerServiceAccountName = "calico-apiserver" - // The API server this one replaces. It shares the calico-apiserver-access-calico-crds binding, - // so while the cutover is held both service accounts must be subjects of it. + // The API server this one replaces. Both service accounts share the + // calico-apiserver-access-calico-crds binding while the cutover is held. deprecatedAPIServerServiceAccountName = "tigera-apiserver" deprecatedAPIServerNamespace = "tigera-system" - APIServerSecretsRBACName = "calico-extension-apiserver-secrets-access" - APIServerLinseedAccessClusterRoleName = "calico-apiserver-linseed-access" - MultiTenantManagedClustersAccessClusterRoleName = "calico-managed-cluster-access" - ManagedClustersWatchClusterRoleName = "calico-managed-cluster-watch" - L7AdmissionControllerContainerName ContainerName = "calico-l7-admission-controller" - L7AdmissionControllerPort = 6443 - L7AdmissionControllerPortName = "l7admctrl" + APIServerSecretsRBACName = "calico-extension-apiserver-secrets-access" + MultiTenantManagedClustersAccessClusterRoleName = "calico-managed-cluster-access" + ManagedClustersWatchClusterRoleName = "calico-managed-cluster-watch" + L7AdmissionControllerContainerName = "calico-l7-admission-controller" + L7AdmissionControllerPort = 6443 + L7AdmissionControllerPortName = "l7admctrl" ) var ( @@ -124,14 +114,29 @@ func APIServer(cfg *APIServerConfiguration) (Component, error) { }, nil } +// apiServerPolicyComponent carries the config a variant needs to layer its own rules +// onto the base policy. +type apiServerPolicyComponent struct { + Component + + cfg *APIServerConfiguration +} + +func (c apiServerPolicyComponent) APIServerPolicyConfig() *APIServerConfiguration { + return c.cfg +} + func APIServerPolicy(cfg *APIServerConfiguration) Component { - return NewPassthrough( - []client.Object{calicoSystemAPIServerPolicy(cfg)}, - []client.Object{ - // allow-tigera Tier was renamed to calico-system - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("apiserver-access", APIServerNamespace), - }, - ) + return apiServerPolicyComponent{ + Component: NewPassthrough( + []client.Object{calicoSystemAPIServerPolicy(cfg)}, + []client.Object{ + // allow-tigera Tier was renamed to calico-system + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("apiserver-access", APIServerNamespace), + }, + ), + cfg: cfg, + } } // APIServerConfiguration contains all the config information needed to render the component. @@ -141,85 +146,44 @@ type APIServerConfiguration struct { Installation *operatorv1.InstallationSpec APIServer *operatorv1.APIServerSpec ForceHostNetwork bool - ApplicationLayer *operatorv1.ApplicationLayer - ManagementCluster *operatorv1.ManagementCluster - ManagementClusterConnection *operatorv1.ManagementClusterConnection TLSKeyPair certificatemanagement.KeyPairInterface PullSecrets []*corev1.Secret OpenShift bool TrustedBundle certificatemanagement.TrustedBundle MultiTenant bool - - // BindingNamespaces are the tenant namespaces whose calico-apiserver ServiceAccount should be granted - // Linseed access on a multi-tenant management cluster. Each managed cluster's calico-apiserver - // authenticates to Linseed with a token whose identity is namespace-overridden to its tenant namespace - // (system:serviceaccount::calico-apiserver), so a ClusterRoleBinding with one subject - // per tenant namespace is required. Empty on zero/single-tenant clusters. - BindingNamespaces []string - - KeyValidatorConfig authentication.KeyValidatorConfig - KubernetesVersion *common.VersionInfo - ClusterDomain string - - // Cloud indicates the API server is being rendered for a Calico Cloud install. It gates - // cloud-specific RBAC in the tigera-ui-user / tigera-network-admin cluster roles (Calico Cloud - // exposes only per-user UISettings and grants access to runtime logs). When false the RBAC is - // exactly the regular Calico/Calico Enterprise RBAC. - Cloud bool - - // RBACManagementEnabled reports whether to render the RBAC management UI access. - // The controller has already applied the variant, the admin's gate and tenancy. - RBACManagementEnabled bool + KubernetesVersion *common.VersionInfo + ClusterDomain string // Whether or not we should run the aggregation API server for projectcalico.org/v3 APIs // as part of this component. RequiresAggregationServer bool - // HoldAPIServiceCutover leaves the previous API server in service: the v3.projectcalico.org - // APIService keeps pointing at it and none of the resources it needs are removed. Set while the - // API server that would take over is not yet ready to serve. + // HoldAPIServiceCutover leaves the previous API server in service, so its + // APIService and the resources it needs are left alone. HoldAPIServiceCutover bool - - // When certificate management is enabled, we need a separate init container to create a cert, running - // with the same permissions as query server. - QueryServerTLSKeyPairCertificateManagementOnly certificatemanagement.KeyPairInterface } type apiServerComponent struct { - cfg *APIServerConfiguration - calicoImage string - l7AdmissionControllerEnvoyImage string - dikastesImage string + cfg *APIServerConfiguration + calicoImage string +} + +func (c *apiServerComponent) APIServerConfig() *APIServerConfiguration { + return c.cfg } func (c *apiServerComponent) ResolveImages(is *operatorv1.ImageSet) error { reg := c.cfg.Installation.Registry path := c.cfg.Installation.ImagePath prefix := c.cfg.Installation.ImagePrefix - var err error - errMsgs := []string{} - - enterprise := c.cfg.Installation.Variant.IsEnterprise() - if enterprise || c.cfg.RequiresAggregationServer { - c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) - if err != nil { - errMsgs = append(errMsgs, err.Error()) - } - } - - if enterprise && c.cfg.IsSidecarInjectionEnabled() { - c.l7AdmissionControllerEnvoyImage, err = components.GetReference(components.ComponentEnvoyProxy, reg, path, prefix, is) - if err != nil { - errMsgs = append(errMsgs, err.Error()) - } - c.dikastesImage, err = components.GetReference(components.ComponentDikastes, reg, path, prefix, is) - if err != nil { - errMsgs = append(errMsgs, err.Error()) - } - } - if len(errMsgs) != 0 { - return fmt.Errorf("%s", strings.Join(errMsgs, ",")) + // Resolve the calico image unconditionally: the base uses it for the aggregation + // API server container, and a variant modifier needs it for the query server + // container and the deployment skeleton it may render itself. + var err error + c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) + if err != nil { + return err } return nil } @@ -229,8 +193,8 @@ func (c *apiServerComponent) SupportedOSType() rmeta.OSType { } func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { - // Start with all of the cluster-scoped resources that are used for both Calico and Calico Enterprise. - // When switching between Calico / Enterprise, these objects are simply updated in-place. + // Cluster-scoped resources used by the API server, independent of variant. Any + // variant-specific objects are layered on by the variant's modifier. globalObjects := []client.Object{ c.calicoCustomResourcesClusterRole(), c.calicoCustomResourcesClusterRoleBinding(), @@ -242,31 +206,20 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { } objsToDelete := []client.Object{} - - // Namespaced objects common to both Calico and Calico Enterprise. - // These objects will be updated when switching between the variants. namespacedObjects := []client.Object{} // Add in image pull secrets. secrets := secret.CopyToNamespace(APIServerNamespace, c.cfg.PullSecrets...) namespacedObjects = append(namespacedObjects, secret.ToRuntimeObjects(secrets...)...) - // The deployment and its supporting objects are needed when running the aggregation API server - // or when running Enterprise (which always needs the queryserver). - if c.cfg.RequiresAggregationServer || c.cfg.Installation.Variant.IsEnterprise() { - namespacedObjects = append(namespacedObjects, - c.apiServerServiceAccount(), - c.apiServerDeployment(), - c.apiServerService(), - c.apiServerPodDisruptionBudget(), - ) + // The deployment and its supporting objects are needed when running the aggregation + // API server. A variant that needs the deployment without an aggregation server (e.g. + // to run only a query server) renders the skeleton itself in its modifier and pulls + // these back out of the delete list. + if c.cfg.RequiresAggregationServer { + namespacedObjects = append(namespacedObjects, c.deploymentObjects()...) } else { - objsToDelete = append(objsToDelete, - &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceAccountName, Namespace: APIServerNamespace}}, - &appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, - &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceName, Namespace: APIServerNamespace}}, - &policyv1.PodDisruptionBudget{TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, - ) + objsToDelete = append(objsToDelete, APIServerDeploymentObjectMeta()...) } // These are objects that only need to exist when we are running an aggregation API server to @@ -279,16 +232,6 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { c.authReaderRoleBinding(), } - if c.cfg.Installation.Variant.IsEnterprise() { - aggregationAPIServerObjects = append(aggregationAPIServerObjects, - c.uiSettingsGroupGetterClusterRole(), - c.kubeControllerManagerUISettingsGroupGetterClusterRoleBinding(), - c.uiSettingsPassthruClusterRole(), - c.uiSettingsPassthruClusterRolebinding(), - c.auditPolicyConfigMap(), - ) - } - // Add in certificates for API server TLS. // // Leaving the APIService unrendered is what holds the cutover, since the component handler only @@ -301,88 +244,11 @@ func (c *apiServerComponent) Objects() ([]client.Object, []client.Object) { } } - // Global enterprise-only objects. - globalEnterpriseObjects := []client.Object{ - c.tigeraAPIServerClusterRole(), - c.tigeraAPIServerClusterRoleBinding(), - } - - if c.cfg.MultiTenant { - // On a multi-tenant management cluster, each tenant's calico-apiserver identity needs read access to - // Linseed. A single least-privilege ClusterRole is bound to the calico-apiserver ServiceAccount in - // every tenant namespace via one ClusterRoleBinding (see linseedAccessClusterRoleBinding). - globalEnterpriseObjects = append(globalEnterpriseObjects, - c.linseedAccessClusterRole(), - c.linseedAccessClusterRoleBinding(), - ) - } else { - // These resources are only installed in zero-tenant clusters. Multi-tenant clusters don't use the default - // RBAC resources. - globalEnterpriseObjects = append(globalEnterpriseObjects, - c.tigeraUserClusterRole(), - c.tigeraNetworkAdminClusterRole(), - ) - // The Linseed-access RBAC is multi-tenant only; ensure it is cleaned up on zero/single-tenant clusters. - objsToDelete = append(objsToDelete, - c.linseedAccessClusterRoleBinding(), - c.linseedAccessClusterRole(), - ) - } - - if c.cfg.ManagementCluster != nil { - globalEnterpriseObjects = append(globalEnterpriseObjects, c.managedClusterWatchClusterRole()) - if c.cfg.MultiTenant { - // Multi-tenant management cluster API servers need access to per-tenant CA secrets in order to sign - // per-tenant guardian certificates when creating ManagedClusters. - globalEnterpriseObjects = append(globalEnterpriseObjects, c.multiTenantSecretsRBAC()...) - // Multi-tenant management cluster components impersonate the single-tenant canonical service account - // in order to retrieve informations from the managed cluster. A cluster role will be created and each - // component will create a role binding in the tenant namespace - globalEnterpriseObjects = append(globalEnterpriseObjects, c.multiTenantManagedClusterAccessClusterRoles()...) - } else { - globalEnterpriseObjects = append(globalEnterpriseObjects, c.secretsRBAC()...) - } - } else { - // If we're not a management cluster, the API server doesn't need permissions to access secrets. - objsToDelete = append(objsToDelete, c.multiTenantSecretsRBAC()...) - objsToDelete = append(objsToDelete, c.secretsRBAC()...) - objsToDelete = append(objsToDelete, c.multiTenantManagedClusterAccessClusterRoles()...) - objsToDelete = append(objsToDelete, c.managedClusterWatchClusterRole()) - } - - // Namespaced enterprise-only objects. - namespacedEnterpriseObjects := []client.Object{} - - if c.cfg.TrustedBundle != nil { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, c.cfg.TrustedBundle.ConfigMap(QueryserverNamespace)) - } - if c.cfg.IsSidecarInjectionEnabled() { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, c.sidecarMutatingWebhookConfig()) - } else { - objsToDelete = append(objsToDelete, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) - } - if c.cfg.ManagementClusterConnection != nil { - namespacedEnterpriseObjects = append(namespacedEnterpriseObjects, - c.externalLinseedRoleBinding(), - ) - } - - // Compile the final arrays based on the variant. - if c.cfg.Installation.Variant.IsEnterprise() { - // Create any enterprise specific objects. - globalObjects = append(globalObjects, globalEnterpriseObjects...) - namespacedObjects = append(namespacedObjects, namespacedEnterpriseObjects...) - - // Clean up cluster-scoped resources that were created with the 'tigera' prefix. - // The apiserver now uses consistent resource names with 'calico' prefix across both EE and OSS variants. - if !c.cfg.HoldAPIServiceCutover { - objsToDelete = append(objsToDelete, c.deprecatedResources()...) - } - } else { - // Explicitly delete any global enterprise objects. - // Namespaced objects will be handled by namespace deletion. - objsToDelete = append(objsToDelete, globalEnterpriseObjects...) - } + // The L7 sidecar mutating webhook is an enterprise (ApplicationLayer) concern added + // by the variant modifier. The base always queues it for deletion so a cluster + // without sidecar injection (including any OSS install) never retains a stale one; + // the modifier pulls it back out of the delete list when sidecar injection is on. + objsToDelete = append(objsToDelete, &admregv1.MutatingWebhookConfiguration{ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}}) // Clean up deprecated k8s NetworkPolicy, regardless of variant, // avoiding leftovers in the case of switching between variants. @@ -413,6 +279,40 @@ func (c *apiServerComponent) Ready() bool { return true } +// deploymentObjects returns the API server Deployment and its supporting objects. +func (c *apiServerComponent) deploymentObjects() []client.Object { + return []client.Object{ + c.apiServerServiceAccount(), + c.apiServerDeployment(), + c.apiServerService(), + c.apiServerPodDisruptionBudget(), + } +} + +// APIServerDeploymentObjects returns the API server Deployment and its supporting +// objects (ServiceAccount, Service, PodDisruptionBudget). The base renders these when +// running an aggregation API server; a variant modifier renders them itself when it +// needs the deployment but the base did not (e.g. a query-server-only deployment in +// v3-CRD mode). calicoImage is the resolved image for any base containers. +func APIServerDeploymentObjects(cfg *APIServerConfiguration, calicoImage string) []client.Object { + c := &apiServerComponent{cfg: cfg, calicoImage: calicoImage} + return c.deploymentObjects() +} + +// APIServerDeploymentObjectMeta returns empty-shell copies of the API server Deployment +// and its supporting objects, identifying them by name/kind/namespace. The base queues +// these for deletion when it isn't running an aggregation API server; a variant modifier +// matches against them to pull them back out of the delete list when it renders the +// deployment skeleton itself. +func APIServerDeploymentObjectMeta() []client.Object { + return []client.Object{ + &corev1.ServiceAccount{TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceAccountName, Namespace: APIServerNamespace}}, + &appsv1.Deployment{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, + &corev1.Service{TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerServiceName, Namespace: APIServerNamespace}}, + &policyv1.PodDisruptionBudget{TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}, ObjectMeta: metav1.ObjectMeta{Name: APIServerName, Namespace: APIServerNamespace}}, + } +} + // For legacy reasons we use apiserver: true here instead of the k8s-app: name label, // so we need to set it explicitly rather than use the common labeling logic. func (c *apiServerComponent) deploymentSelector() *metav1.LabelSelector { @@ -530,38 +430,6 @@ func (c *apiServerComponent) apiServerServiceAccount() *corev1.ServiceAccount { } } -// linseedAccessClusterRole is a minimal, least-privilege ClusterRole granting the calico-apiserver identity -// read access to Linseed policy activity data (for queryserver enrichment). On a multi-tenant management -// cluster it is bound to each tenant's calico-apiserver ServiceAccount so that tenant's managed clusters can -// reach Linseed. The full backing-storage/queryserver rules live on the calico-apiserver ClusterRole, which is -// bound only to the calico-system API server itself. -// -// Calico Enterprise, multi-tenant only. -func (c *apiServerComponent) linseedAccessClusterRole() *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: APIServerLinseedAccessClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - // Read access to Linseed policy activity data for queryserver enrichment. - APIGroups: []string{"linseed.tigera.io"}, - Resources: []string{"policyactivity"}, - Verbs: []string{"get"}, - }, - }, - } -} - -// linseedAccessClusterRoleBinding binds the Linseed-access ClusterRole to the calico-apiserver ServiceAccount -// in each tenant namespace. Linseed authorizes with a cluster-scoped SubjectAccessReview, so this is a single -// ClusterRoleBinding with one ServiceAccount subject per tenant namespace - mirroring how compliance, -// intrusion-detection, and the manager grant their managed-cluster components Linseed access. -// -// Calico Enterprise, multi-tenant only. -func (c *apiServerComponent) linseedAccessClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return rcomp.ClusterRoleBinding(APIServerLinseedAccessClusterRoleName, APIServerLinseedAccessClusterRoleName, APIServerServiceAccountName, c.cfg.BindingNamespaces) -} - func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy { egressRules := []v3.Rule{} egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.OpenShift) @@ -589,13 +457,6 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy }, }...) - if cfg.KeyValidatorConfig != nil { - if parsedURL, err := url.Parse(cfg.KeyValidatorConfig.Issuer()); err == nil { - oidcEgressRule := networkpolicy.GetOIDCEgressRule(parsedURL) - egressRules = append(egressRules, oidcEgressRule) - } - } - if r, err := cfg.K8SServiceEndpoint.DestinationEntityRule(); r != nil && err == nil { egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, @@ -610,15 +471,12 @@ func calicoSystemAPIServerPolicy(cfg *APIServerConfiguration) *v3.NetworkPolicy Action: v3.Pass, }) - apiServerContainerPort := getContainerPort(cfg, APIServerContainerName).ContainerPort - queryServerContainerPort := getContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - l7AdmCtrlContainerPort := getContainerPort(cfg, L7AdmissionControllerContainerName).ContainerPort + apiServerContainerPort := GetContainerPort(cfg, APIServerContainerName).ContainerPort + queryServerContainerPort := GetContainerPort(cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - // The ports Calico Enterprise API Server and Calico Enterprise Query Server are configured to listen on. + // The ports the API server and query server listen on. The L7 admission controller + // ingress port is added by the variant modifier when sidecar injection is enabled. ingressPorts := networkpolicy.Ports(443, uint16(apiServerContainerPort), uint16(queryServerContainerPort), 10443) - if cfg.IsSidecarInjectionEnabled() { - ingressPorts = append(ingressPorts, numorstring.Port{MinPort: uint16(l7AdmCtrlContainerPort), MaxPort: uint16(l7AdmCtrlContainerPort)}) - } return &v3.NetworkPolicy{ TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, @@ -775,9 +633,8 @@ func (c *apiServerComponent) calicoCustomResourcesClusterRoleBinding() *rbacv1.C subjects := []rbacv1.Subject{ {Kind: "ServiceAccount", Name: APIServerServiceAccountName, Namespace: APIServerNamespace}, } - // The previous API server is still serving while the cutover is held, and it reads its own - // storage through this binding, so dropping it here would take the API down. if c.cfg.HoldAPIServiceCutover { + // The previous API server is still serving, and reads its storage through this binding. subjects = append(subjects, rbacv1.Subject{ Kind: "ServiceAccount", Name: deprecatedAPIServerServiceAccountName, @@ -862,17 +719,6 @@ func (c *apiServerComponent) authClusterRole() client.Object { } } -// multiTenantSecretsRBAC provides the tigera API server with the ability to read secrets on the cluster. -// This is needed in multi-tenant management clusters only, in order to read tenant secrets for signing managed cluster certificates. -func (c *apiServerComponent) multiTenantSecretsRBAC() []client.Object { - return TunnelSecretRBAC(APIServerSecretsRBACName, APIServerServiceAccountName, c.cfg.ManagementCluster, true) -} - -// secretsRBAC provides the tigera API server with the ability to read secrets from the API server's namespace. -func (c *apiServerComponent) secretsRBAC() []client.Object { - return TunnelSecretRBAC(APIServerSecretsRBACName, APIServerServiceAccountName, c.cfg.ManagementCluster, false) -} - // authClusterRoleBinding returns a clusterrolebinding to create, and a clusterrolebinding to delete. // // Both Calico and Calico Enterprise, with different names. @@ -948,7 +794,7 @@ func (c *apiServerComponent) webhookReaderClusterRoleBinding() client.Object { } } -func getContainerPort(cfg *APIServerConfiguration, containerName ContainerName) *operatorv1.APIServerDeploymentContainerPort { +func GetContainerPort(cfg *APIServerConfiguration, containerName string) *operatorv1.APIServerDeploymentContainerPort { // Try to get the override port if cfg != nil && cfg.APIServer != nil && @@ -983,11 +829,10 @@ func getContainerPort(cfg *APIServerConfiguration, containerName ContainerName) return nil } -// apiServerService creates a service backed by the API server and - for enterprise - query server. +// apiServerService creates a service backed by the API server. A variant modifier may add +// additional ports (e.g. the query server port). func (c *apiServerComponent) apiServerService() *corev1.Service { - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName) - queryServerTargetPort := getContainerPort(c.cfg, TigeraAPIServerQueryServerContainerName) - l7AdmissionControllerTargetPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName) + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName) s := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, @@ -1011,29 +856,6 @@ func (c *apiServerComponent) apiServerService() *corev1.Service { }, } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add port for queryserver if enterprise. - s.Spec.Ports = append(s.Spec.Ports, - corev1.ServicePort{ - Name: QueryServerPortName, - Port: QueryServerPort, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt32(queryServerTargetPort.ContainerPort), - }, - ) - } - - if c.cfg.IsSidecarInjectionEnabled() { - s.Spec.Ports = append(s.Spec.Ports, - corev1.ServicePort{ - Name: L7AdmissionControllerPortName, - Port: L7AdmissionControllerPort, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromInt32(l7AdmissionControllerTargetPort.ContainerPort), - }, - ) - } - return s } @@ -1052,31 +874,22 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { c.cfg.TLSKeyPair.HashAnnotationKey(): c.cfg.TLSKeyPair.HashAnnotationValue(), } + // The API server cert init container is only needed when running the aggregation API + // server under certificate management. A variant modifier adds any further init + // containers it needs (e.g. the query server cert). var initContainers []corev1.Container - if c.cfg.TLSKeyPair.UseCertificateManagement() { - if c.cfg.RequiresAggregationServer { - // Only include the API server init container if we're running the aggregation API server! - initContainerAPIServer := c.cfg.TLSKeyPair.InitContainer(APIServerNamespace, c.apiServerContainer().SecurityContext) - initContainerAPIServer.Name = fmt.Sprintf("%s-%s", CalicoAPIServerTLSSecretName, certificatemanagement.CSRInitContainerName) - initContainers = append(initContainers, initContainerAPIServer) - } - - initContainerQueryServer := c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.InitContainer(APIServerNamespace, c.queryServerContainer().SecurityContext) - annotations[c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationKey()] = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.HashAnnotationValue() - initContainers = append(initContainers, initContainerQueryServer) + if c.cfg.TLSKeyPair.UseCertificateManagement() && c.cfg.RequiresAggregationServer { + initContainerAPIServer := c.cfg.TLSKeyPair.InitContainer(APIServerNamespace, c.apiServerContainer().SecurityContext) + initContainerAPIServer.Name = fmt.Sprintf("%s-%s", CalicoAPIServerTLSSecretName, certificatemanagement.CSRInitContainerName) + initContainers = append(initContainers, initContainerAPIServer) } - // Determine which containers to run. + // Determine which containers to run. A variant modifier may add additional + // containers (e.g. the query server and the L7 admission controller). containers := []corev1.Container{} if c.cfg.RequiresAggregationServer { containers = append(containers, c.apiServerContainer()) } - if c.cfg.IsSidecarInjectionEnabled() { - containers = append(containers, c.l7AdmissionControllerContainer()) - } - if c.cfg.Installation.Variant.IsEnterprise() { - containers = append(containers, c.queryServerContainer()) - } d := &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, @@ -1121,15 +934,6 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { d.Spec.Template.Spec.Affinity = podaffinity.NewPodAntiAffinity(APIServerName, []string{APIServerNamespace, "tigera-system", "calico-apiserver"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.TrustedBundle != nil { - trustedBundleHashAnnotations := c.cfg.TrustedBundle.HashAnnotations() - for k, v := range trustedBundleHashAnnotations { - d.Spec.Template.Annotations[k] = v - } - } - } - if overrides := c.cfg.APIServer.APIServerDeployment; overrides != nil { rcomp.ApplyDeploymentOverrides(d, overrides) } @@ -1137,70 +941,17 @@ func (c *apiServerComponent) apiServerDeployment() *appsv1.Deployment { return d } -// apiServer creates a MutatingWebhookConfiguration for sidecars. -func (c *apiServerComponent) sidecarMutatingWebhookConfig() *admregv1.MutatingWebhookConfiguration { - var cacert []byte - svcPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort - - svcpath := "/sidecar-webhook" - svcref := admregv1.ServiceReference{ - Name: QueryserverServiceName, - Namespace: QueryserverNamespace, - Path: &svcpath, - Port: &svcPort, - } - failpol := admregv1.Fail - labelsel := metav1.LabelSelector{ - MatchLabels: map[string]string{ - "applicationlayer.projectcalico.org/sidecar": "true", - }, - } - rules := []admregv1.RuleWithOperations{ - { - Rule: admregv1.Rule{ - APIGroups: []string{""}, - APIVersions: []string{"v1"}, - Resources: []string{"pods"}, - }, - Operations: []admregv1.OperationType{admregv1.Create}, - }, - } - sidefx := admregv1.SideEffectClassNone - if !c.cfg.TLSKeyPair.UseCertificateManagement() { - cacert = c.cfg.TLSKeyPair.GetIssuer().GetCertificatePEM() - } else { - cacert = c.cfg.Installation.CertificateManagement.CACert - } - mwc := admregv1.MutatingWebhookConfiguration{ - TypeMeta: metav1.TypeMeta{ - Kind: "MutatingWebhookConfiguration", - APIVersion: "admissionregistration.k8s.io/v1", - }, - ObjectMeta: metav1.ObjectMeta{Name: common.SidecarMutatingWebhookConfigName}, - Webhooks: []admregv1.MutatingWebhook{ - { - AdmissionReviewVersions: []string{"v1"}, - ClientConfig: admregv1.WebhookClientConfig{ - Service: &svcref, - CABundle: cacert, - }, - Name: "sidecar.projectcalico.org", - FailurePolicy: &failpol, - ObjectSelector: &labelsel, - Rules: rules, - SideEffects: &sidefx, - }, - }, - } - - return &mwc +func (c *apiServerComponent) hostNetwork() bool { + return HostNetwork(c.cfg) } -func (c *apiServerComponent) hostNetwork() bool { - if c.cfg.ForceHostNetwork { +// HostNetwork reports whether the API server deployment runs on the host network, +// accounting for both the forced setting and the provider-driven requirement. +func HostNetwork(cfg *APIServerConfiguration) bool { + if cfg.ForceHostNetwork { return true } - return HostNetworkRequired(c.cfg.Installation) + return HostNetworkRequired(cfg.Installation) } func HostNetworkRequired(installation *operatorv1.InstallationSpec) bool { @@ -1220,12 +971,6 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { volumeMounts := []corev1.VolumeMount{ c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), } - if c.cfg.Installation.Variant.IsEnterprise() { - volumeMounts = append(volumeMounts, - corev1.VolumeMount{Name: auditLogsVolumeName, MountPath: "/var/log/calico/audit"}, - corev1.VolumeMount{Name: auditPolicyVolumeName, MountPath: "/etc/tigera/audit"}, - ) - } env := []corev1.EnvVar{ {Name: "DATASTORE_TYPE", Value: "kubernetes"}, @@ -1255,10 +1000,10 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) } - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName).ContainerPort + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName).ContainerPort apiServer := corev1.Container{ - Name: string(APIServerContainerName), + Name: APIServerContainerName, Image: c.calicoImage, Command: []string{components.CalicoBinaryPath, "component", "apiserver"}, Args: c.startUpArgs(), @@ -1277,19 +1022,13 @@ func (c *apiServerComponent) apiServerContainer() corev1.Container { PeriodSeconds: 60, }, } - // In case of OpenShift, apiserver needs privileged access to write audit logs to host path volume. - // Audit logs are owned by root on hosts so we need to be root user and group. Audit logs are supported only in Enterprise version. - if c.cfg.Installation.Variant.IsEnterprise() { - apiServer.SecurityContext = securitycontext.NewRootContext(c.cfg.OpenShift) - } else { - apiServer.SecurityContext = securitycontext.NewNonRootContext() - } + apiServer.SecurityContext = securitycontext.NewNonRootContext() return apiServer } func (c *apiServerComponent) startUpArgs() []string { - apiServerTargetPort := getContainerPort(c.cfg, APIServerContainerName).ContainerPort + apiServerTargetPort := GetContainerPort(c.cfg, APIServerContainerName).ContainerPort args := []string{ fmt.Sprintf("--secure-port=%d", apiServerTargetPort), @@ -1297,25 +1036,9 @@ func (c *apiServerComponent) startUpArgs() []string { fmt.Sprintf("--tls-cert-file=%s", c.cfg.TLSKeyPair.VolumeMountCertificateFilePath()), } - if c.cfg.Installation.Variant.IsEnterprise() { - args = append(args, - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - ) - } - - if c.cfg.ManagementCluster != nil { - args = append(args, "--enable-managed-clusters-create-api=true") - if c.cfg.ManagementCluster.Spec.Address != "" { - args = append(args, fmt.Sprintf("--managementClusterAddr=%s", c.cfg.ManagementCluster.Spec.Address)) - } - if c.cfg.ManagementCluster.Spec.TLS != nil && c.cfg.ManagementCluster.Spec.TLS.SecretName != "" { - if c.cfg.ManagementCluster.Spec.TLS.SecretName == ManagerTLSSecretName { - args = append(args, "--managementClusterCAType=Public") - } - args = append(args, fmt.Sprintf("--tunnelSecretName=%s", c.cfg.ManagementCluster.Spec.TLS.SecretName)) - } - } + // The management-cluster tunnel args (--enable-managed-clusters-create-api, + // --managementClusterAddr, --tunnelSecretName, --managementClusterCAType) are an + // enterprise concern, appended to this container by the variant modifier. if c.cfg.KubernetesVersion != nil && c.cfg.KubernetesVersion.Major < 2 && c.cfg.KubernetesVersion.Minor < 30 { // Disable this API as it is not available by default. If we don't, the server fails to start, due to trying to // establish watches for unavailable APIs. @@ -1324,1154 +1047,114 @@ func (c *apiServerComponent) startUpArgs() []string { return args } -// queryServerContainer creates the query server container. -func (c *apiServerComponent) queryServerContainer() corev1.Container { - queryServerTargetPort := getContainerPort(c.cfg, TigeraAPIServerQueryServerContainerName).ContainerPort - - var tlsSecret certificatemanagement.KeyPairInterface - if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { - tlsSecret = c.cfg.QueryServerTLSKeyPairCertificateManagementOnly - } else { - tlsSecret = c.cfg.TLSKeyPair - } - env := []corev1.EnvVar{ - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "LISTEN_ADDR", Value: fmt.Sprintf(":%d", queryServerTargetPort)}, - {Name: "TLS_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, - {Name: "TLS_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "TRUSTED_BUNDLE_PATH", Value: c.cfg.TrustedBundle.MountPath()}) +// apiServerVolumes creates the volumes used by the API server deployment. A variant +// modifier adds any further volumes it needs (the query server cert and the Linseed +// token). +func (c *apiServerComponent) apiServerVolumes() []corev1.Volume { + return []corev1.Volume{ + c.cfg.TLSKeyPair.Volume(), } +} +// tolerations creates the tolerations used by the API server deployment. +func (c *apiServerComponent) tolerations() []corev1.Toleration { if c.hostNetwork() { - env = append(env, c.cfg.K8SServiceEndpoint.EnvVars()...) - } else { - env = append(env, c.cfg.K8SServiceEndpointPodNetwork.EnvVars()...) - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if c.cfg.KeyValidatorConfig != nil { - env = append(env, c.cfg.KeyValidatorConfig.RequiredEnv("")...) - } - - linseedURL := relasticsearch.LinseedEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, ElasticsearchNamespace, c.cfg.ManagementClusterConnection != nil, false) - env = append(env, - corev1.EnvVar{Name: "LINSEED_URL", Value: linseedURL}, - corev1.EnvVar{Name: "LINSEED_CLIENT_CERT", Value: fmt.Sprintf("/%s/tls.crt", tlsSecret.GetName())}, - corev1.EnvVar{Name: "LINSEED_CLIENT_KEY", Value: fmt.Sprintf("/%s/tls.key", tlsSecret.GetName())}, - ) - if c.cfg.ManagementClusterConnection != nil { - env = append(env, - corev1.EnvVar{Name: "CLUSTER_ID", Value: ""}, - corev1.EnvVar{Name: "LINSEED_TOKEN", Value: GetLinseedTokenPath(true)}, - ) - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "LINSEED_CA", Value: c.cfg.TrustedBundle.MountPath()}) - } - - // set LogLEVEL for queryserver container - if logging := c.cfg.APIServer.Logging; logging != nil && - logging.QueryServerLogging != nil && logging.QueryServerLogging.LogSeverity != nil { - env = append(env, - corev1.EnvVar{Name: "LOGLEVEL", Value: strings.ToLower(string(*logging.QueryServerLogging.LogSeverity))}) - } else { - // set default LOGLEVEL to info when not set by the user - env = append(env, corev1.EnvVar{Name: "LOGLEVEL", Value: "info"}) - } - - volumeMounts := []corev1.VolumeMount{ - tlsSecret.VolumeMount(c.SupportedOSType()), - } - if c.cfg.TrustedBundle != nil { - volumeMounts = append(volumeMounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) + return rmeta.TolerateBootstrap } - if c.cfg.ManagementClusterConnection != nil { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: LinseedTokenVolumeName, - MountPath: LinseedVolumeMountPath, - }) + tolerations := append(c.cfg.Installation.ControlPlaneTolerations, rmeta.TolerateControlPlane...) + if c.cfg.Installation.KubernetesProvider.IsGKE() { + tolerations = append(tolerations, rmeta.TolerateGKEARM64NoSchedule) } + return tolerations +} - container := corev1.Container{ - Name: string(TigeraAPIServerQueryServerContainerName), - Image: c.calicoImage, - Command: []string{components.CalicoBinaryPath, "component", "queryserver"}, - Env: env, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/version", - Port: intstr.FromInt32(queryServerTargetPort), - Scheme: corev1.URISchemeHTTPS, +// tierGetterClusterRole creates a clusterrole that gives permissions to get tiers. +func (c *apiServerComponent) tierGetterClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "calico-tier-getter", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "tiers", }, + Verbs: []string{"get"}, }, - InitialDelaySeconds: 90, }, - SecurityContext: securitycontext.NewNonRootContext(), - VolumeMounts: volumeMounts, } - return container } -func (c *apiServerComponent) externalLinseedRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, +// kubeControllerMgrTierGetterClusterRoleBinding creates a rolebinding that allows the k8s kube-controller manager to +// get tiers. In k8s 1.15+, cascading resource deletions (for instance pods for a replicaset) failed +// due to k8s kube-controller not having permissions to get tiers. +func (c *apiServerComponent) kubeControllerMgrTierGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-linseed", - Namespace: APIServerNamespace, + Name: "calico-tier-getter", }, RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", - Name: TigeraLinseedSecretsClusterRole, + Name: "calico-tier-getter", + APIGroup: "rbac.authorization.k8s.io", }, Subjects: []rbacv1.Subject{ { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, + Kind: "User", + Name: "system:kube-controller-manager", + APIGroup: "rbac.authorization.k8s.io", }, }, } } -// apiServerVolumes creates the volumes used by the API server deployment. -func (c *apiServerComponent) apiServerVolumes() []corev1.Volume { - volumes := []corev1.Volume{ - c.cfg.TLSKeyPair.Volume(), - } - if c.cfg.QueryServerTLSKeyPairCertificateManagementOnly != nil { - volumes = append(volumes, c.cfg.QueryServerTLSKeyPairCertificateManagementOnly.Volume()) - } +// calicoPolicyPassthruClusterRole creates a clusterrole that is used to control the RBAC +// mechanism for Calico tiered policy. +func (c *apiServerComponent) calicoPolicyPassthruClusterRole() *rbacv1.ClusterRole { + resources := []string{"networkpolicies", "globalnetworkpolicies"} - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.RequiresAggregationServer { - // Only include these volumes if we're running the aggregation API server, since audit logging is done through the - // main API server otherwise. - volumes = append(volumes, - corev1.Volume{ - Name: auditLogsVolumeName, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/log/calico/audit", - Type: ptr.To(corev1.HostPathDirectoryOrCreate), - }, - }, - }, - corev1.Volume{ - Name: auditPolicyVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: auditPolicyVolumeName}, - Items: []corev1.KeyToPath{ - { - Key: "config", - Path: "policy.conf", - }, - }, - }, - }, + return &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: TieredPolicyPassthruClusterRoleName, + }, + // If tiered policy is enabled we allow all authenticated users to access the main tier resource, instead + // restricting access using the tier.xxx resource type. Kubernetes NetworkPolicy and + // StagedKubernetesNetworkPolicy objects are handled using normal (non-tiered) RBAC. + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"projectcalico.org"}, + Resources: resources, + Verbs: allVerbs, }, - ) + }, } +} - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.TrustedBundle != nil { - volumes = append(volumes, c.cfg.TrustedBundle.Volume()) +// calicoPolicyPassthruClusterRolebinding creates a clusterrolebinding that applies calicoPolicyPassthruClusterRole to all users. +func (c *apiServerComponent) calicoPolicyPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: TieredPolicyPassthruClusterRoleName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "Group", + Name: "system:authenticated", + APIGroup: "rbac.authorization.k8s.io", + }, + }, + RoleRef: rbacv1.RoleRef{ + Kind: "ClusterRole", + Name: TieredPolicyPassthruClusterRoleName, + APIGroup: "rbac.authorization.k8s.io", + }, } - - if c.cfg.ManagementClusterConnection != nil { - // Optional: the Secret is delivered over the Guardian tunnel, which can't be - // established until calico-apiserver is Ready. - volumes = append(volumes, corev1.Volume{ - Name: LinseedTokenVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf(LinseedTokenSecret, "calico-apiserver"), - Items: []corev1.KeyToPath{{Key: LinseedTokenKey, Path: LinseedTokenSubPath}}, - Optional: ptr.To(true), - }, - }, - }) - } - - return volumes -} - -// tolerations creates the tolerations used by the API server deployment. -func (c *apiServerComponent) tolerations() []corev1.Toleration { - if c.hostNetwork() { - return rmeta.TolerateBootstrap - } - tolerations := append(c.cfg.Installation.ControlPlaneTolerations, rmeta.TolerateControlPlane...) - if c.cfg.Installation.KubernetesProvider.IsGKE() { - tolerations = append(tolerations, rmeta.TolerateGKEARM64NoSchedule) - } - return tolerations -} - -// tigeraAPIServerClusterRole creates a clusterrole that gives permissions to access backing CRDs -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraAPIServerClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - { - // Read access to Linseed policy activity data for queryserver enrichment. - APIGroups: []string{"linseed.tigera.io"}, - Resources: []string{"policyactivity"}, - Verbs: []string{"get"}, - }, - { - // Calico Enterprise backing storage. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "bfdconfigurations", - "deeppacketinspections", - "deeppacketinspections/status", - "egressgatewaypolicies", - "externalnetworks", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalreports", - "globalreports/status", - "globalreporttypes", - "globalthreatfeeds", - "globalthreatfeeds/status", - "licensekeys", - "managedclusters", - "managedclusters/status", - "networks", - "packetcaptures", - "packetcaptures/status", - "policyrecommendationscopes", - "policyrecommendationscopes/status", - "remoteclusterconfigurations", - "securityeventwebhooks", - "securityeventwebhooks/status", - "uisettings", - "uisettingsgroups", - }, - Verbs: []string{ - "get", - "list", - "watch", - "create", - "update", - "delete", - "patch", - }, - }, - { - // The queryserver's RBAC calculator needs to list tiers, - // uisettingsgroups, and managedclusters via the aggregated - // API to evaluate user permissions for the /policies endpoint. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "tiers", - "uisettingsgroups", - "managedclusters", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Required by the AuthorizationReview calculator in queryserver to evaluate - // RBAC permissions for users. - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{ - "clusterroles", - "clusterrolebindings", - "roles", - "rolebindings", - }, - Verbs: []string{"get", "list", "watch"}, - }, - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: APIServerName, - }, - Rules: rules, - } -} - -// tigeraAPIServerClusterRoleBinding creates a clusterrolebinding that applies tigeraAPIServerClusterRole to -// the calico-apiserver service account. This is the aggregated API server's own identity, which always runs -// in calico-system - including on multi-tenant management clusters, where the API server is a single -// cluster-scoped component rather than a per-tenant one. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraAPIServerClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: APIServerName, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: APIServerServiceAccountName, - Namespace: APIServerNamespace, - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: APIServerName, - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - -// tierGetterClusterRole creates a clusterrole that gives permissions to get tiers. -func (c *apiServerComponent) tierGetterClusterRole() *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-tier-getter", - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "tiers", - }, - Verbs: []string{"get"}, - }, - }, - } -} - -// kubeControllerMgrTierGetterClusterRoleBinding creates a rolebinding that allows the k8s kube-controller manager to -// get tiers. In k8s 1.15+, cascading resource deletions (for instance pods for a replicaset) failed -// due to k8s kube-controller not having permissions to get tiers. -func (c *apiServerComponent) kubeControllerMgrTierGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-tier-getter", - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-tier-getter", - APIGroup: "rbac.authorization.k8s.io", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "User", - Name: "system:kube-controller-manager", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - } -} - -// uiSettingsGroupGetterClusterRole creates a clusterrole that gives permissions to get uisettingsgroups. -// -// Calico Enterprise only -func (c *apiServerComponent) uiSettingsGroupGetterClusterRole() *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettingsgroup-getter", - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "uisettingsgroups", - }, - Verbs: []string{"get"}, - }, - }, - } -} - -// kubeControllerManagerUISettingsGroupGetterClusterRoleBinding creates a rolebinding that allows the k8s kube-controller -// manager to get uisettingsgroups. -// -// In k8s 1.15+, cascading resource deletions (for instance pods for a replicaset) failed due to k8s kube-controller -// not having permissions to get tiers. UISettings and UISettingsGroups RBAC works in a similar way to tiered policy -// and so we need similar RBAC for UISettingsGroups. -// -// Calico Enterprise only -func (c *apiServerComponent) kubeControllerManagerUISettingsGroupGetterClusterRoleBinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettingsgroup-getter", - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-uisettingsgroup-getter", - APIGroup: "rbac.authorization.k8s.io", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "User", - Name: "system:kube-controller-manager", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - } -} - -// tigeraUserClusterRole returns a cluster role for a default Calico Enterprise user. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraUserClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - // List requests that the Tigera manager needs. - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - "", - }, - // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior - // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for - // more details). Similar for all tiered policy resource types. - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "namespaces", - "globalnetworksets", - "networksets", - "managedclusters", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "policyrecommendationscopes", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to view Networks. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Additional "list" requests required to view flows. - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Additional "list" requests required to view serviceaccount labels. - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - // Access for WAF API to read in coreruleset configmap - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - // Access to statistics. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - // Access to policies in all tiers - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"get"}, - }, - // List and download the reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - // Access to hostendpoints from the UI ServiceGraph. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - // List and view the threat defense configuration - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"get", "watch", "list"}, - }, - } - - // User can: - // - read UISettings in the cluster-settings group (non-cloud only) - // - read and write UISettings in the user-settings group - // Default settings group and settings are created in manager.go. - // Calico Cloud exposes only per-user UISettings, so the cluster-settings group is omitted there. - if c.cfg.Cloud { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"user-settings"}, - }, - ) - } else { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"get", "list", "watch"}, - ResourceNames: []string{"cluster-settings"}, - }, - ) - } - - rules = append(rules, []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"user-settings"}, - }, - // Allow the user to read applicationlayers to detect if WAF is enabled/disabled. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get"}, - }, - // Allow the user to read the gatewayapis CR to detect if Gateway API is enabled/disabled. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"gatewayapis"}, - Verbs: []string{"get"}, - }, - // Allow the user to read Gateways and HTTPRoutes to offer as WAF policy attach targets. - { - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to view WAF policies, plugins, and validation policies. - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "globalwafpolicies", - "globalwafplugins", - "globalwafvalidationpolicies", - "wafpolicies", - "wafplugins", - "wafvalidationpolicies", - }, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to read services to view WAF configuration. - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - // Allow the user to only view securityeventwebhooks. - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list"}, - }, - }...) - - // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { - // Access to flow logs, audit logs, and statistics. - // Access to log into Kibana for oidc users. - // Calico Cloud additionally grants access to runtime logs. - resourceNames := []string{"flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations"} - if c.cfg.Cloud { - resourceNames = append([]string{"runtime"}, resourceNames...) - } - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: resourceNames, - Verbs: []string{"get"}, - }) - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-ui-user", - }, - Rules: rules, - } -} - -// tigeraNetworkAdminClusterRole returns a cluster role for a Tigera Secure manager network admin. -// -// Calico Enterprise only -func (c *apiServerComponent) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole { - rules := []rbacv1.PolicyRule{ - // Full access to all network policies - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - }, - // Use both the networkpolicies and tier.networkpolicies resource types to ensure identical behavior - // irrespective of the Calico RBAC scheme (see the ClusterRole "calico-tiered-policy-passthrough" for - // more details). Similar for all tiered policy resource types. - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "globalnetworksets", - "networksets", - "managedclusters", - "packetcaptures", - "policyrecommendationscopes", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{ - "policy.networking.k8s.io", - }, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get", "delete"}, - }, - // Allow the user to CRUD Networks. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - // Additional "list" requests that the Tigera Secure manager needs - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"watch", "list"}, - }, - // Additional "list" requests required to view flows. - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Additional "list" requests required to view serviceaccount labels. - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - // Access for WAF API to read in coreruleset configmap - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - // Access to statistics. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - // Manage globalreport configuration, view report generation status, and list reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"*"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports/status"}, - Verbs: []string{"get", "list", "watch"}, - }, - // List and download the reports in the Tigera Secure manager. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - // Access to cluster information containing Calico and EE versions from the UI. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - // Access to hostendpoints from the UI ServiceGraph. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - // Manage the threat defense configuration - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - } - - // User can: - // - read and write UISettings in the cluster-settings group, and rename the group (non-cloud only) - // - read and write UISettings in the user-settings group, and rename the group - // Default settings group and settings are created in manager.go. - // Calico Cloud exposes only per-user UISettings, so the cluster-settings group is omitted there. - if c.cfg.Cloud { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"user-settings"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"user-settings"}, - }, - ) - } else { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get", "patch", "update"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - ) - } - - rules = append(rules, []rbacv1.PolicyRule{ - // Allow the user to read and write applicationlayers to enable/disable WAF. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get", "update", "patch", "create", "delete"}, - }, - // Allow the user to read the gatewayapis CR to detect if Gateway API is enabled/disabled. - // Read-only: enabling Gateway API from the WAF UI is intentionally deferred. - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"gatewayapis"}, - Verbs: []string{"get"}, - }, - // Allow the user to read Gateways and HTTPRoutes to offer as WAF policy attach targets. - { - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Allow the user to manage WAF policies, plugins, and validation policies. - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "globalwafpolicies", - "globalwafplugins", - "globalwafvalidationpolicies", - "wafpolicies", - "wafplugins", - "wafvalidationpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - // Allow the user to read deployments to view WAF configuration. - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - // Allow the user to read felixconfigurations to detect if wireguard and/or other features are enabled. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - // Allow the user to perform CRUD operations on securityeventwebhooks. - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, - }, - // Allow the user to create secrets. - { - APIGroups: []string{""}, - Resources: []string{ - "secrets", - }, - Verbs: []string{"create"}, - }, - // Allow the user to patch webhooks-secret secret. - { - APIGroups: []string{""}, - Resources: []string{ - "secrets", - }, - ResourceNames: []string{ - "webhooks-secret", - }, - Verbs: []string{"patch"}, - }, - }...) - - // Write access to the switch, so a network admin can enable the feature without - // cluster-admin. Not gated: a rule rendered only while the feature is on could never - // be used to turn it on. create cannot be restricted by resource name, so it admits - // creating any ConfigMap in the namespace. - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"create"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{rbacmanagement.ConfigMapName}, - Verbs: []string{"get", "list", "watch", "update", "patch", "delete"}, - }, - ) - - // Role/binding access for the RBAC management UI. ui-apis writes these impersonating - // the caller, so the apiserver enforces escalation against the user's own permissions. - if c.cfg.RBACManagementEnabled { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "roles"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterrolebindings", "rolebindings"}, - Verbs: []string{"get", "list", "watch", "create", "update", "delete"}, - }, - ) - } - - // Privileges for lma.tigera.io have no effect on managed clusters. - if c.cfg.ManagementClusterConnection == nil { - // Access to flow logs, audit logs, and statistics. - // Elasticsearch superuser access once logged into Kibana. - // Calico Cloud additionally grants access to runtime logs. - resourceNames := []string{"flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations"} - if c.cfg.Cloud { - resourceNames = append([]string{"runtime"}, resourceNames...) - } - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: resourceNames, - Verbs: []string{"get"}, - }) - } - - // In v3 CRD / webhooks mode there is no aggregated apiserver, and the - // calico-uisettings-passthrough ClusterRole that normally grants the broad - // uisettings permission isn't deployed. Grant write verbs here so the - // calico-webhooks UISettings handler (which narrows access via a SAR on - // uisettingsgroups/data) gets invoked instead of being short-circuited by - // kube-apiserver RBAC. - if !c.cfg.RequiresAggregationServer { - rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"create", "update", "delete", "patch"}, - }) - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-network-admin", - }, - Rules: rules, - } -} - -// calicoPolicyPassthruClusterRole creates a clusterrole that is used to control the RBAC -// mechanism for Calico tiered policy. -func (c *apiServerComponent) calicoPolicyPassthruClusterRole() *rbacv1.ClusterRole { - resources := []string{"networkpolicies", "globalnetworkpolicies"} - - // Append additional resources for enterprise Variant. - if c.cfg.Installation.Variant.IsEnterprise() { - resources = append(resources, "stagednetworkpolicies", "stagedglobalnetworkpolicies") - } - - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-tiered-policy-passthrough", - }, - // If tiered policy is enabled we allow all authenticated users to access the main tier resource, instead - // restricting access using the tier.xxx resource type. Kubernetes NetworkPolicy and - // StagedKubernetesNetworkPolicy objects are handled using normal (non-tiered) RBAC. - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: resources, - Verbs: allVerbs, - }, - }, - } -} - -// calicoPolicyPassthruClusterRolebinding creates a clusterrolebinding that applies calicoPolicyPassthruClusterRole to all users. -func (c *apiServerComponent) calicoPolicyPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-tiered-policy-passthrough", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:authenticated", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-tiered-policy-passthrough", - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - -// uiSettingsPassthruClusterRole creates a clusterrole that is used to control the RBAC mechanism for Tigera UI Settings. -// RBAC for these is handled within the Tigera API Server which checks uisettingsgroups/data permissions for the user. -// -// Calico Enterprise only -func (c *apiServerComponent) uiSettingsPassthruClusterRole() *rbacv1.ClusterRole { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettings-passthrough", - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"*"}, - }, - }, - } -} - -// uiSettingsPassthruClusterRolebinding creates a clusterrolebinding that applies uiSettingsPassthruClusterRole to all -// users. -// -// Calico Enterprise only. -func (c *apiServerComponent) uiSettingsPassthruClusterRolebinding() *rbacv1.ClusterRoleBinding { - return &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "calico-uisettings-passthrough", - }, - Subjects: []rbacv1.Subject{ - { - Kind: "Group", - Name: "system:authenticated", - APIGroup: "rbac.authorization.k8s.io", - }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: "calico-uisettings-passthrough", - APIGroup: "rbac.authorization.k8s.io", - }, - } -} - -// auditPolicyConfigMap returns a configmap with contents to configure audit logging for -// projectcalico.org/v3 APIs. -// -// Calico Enterprise only -func (c *apiServerComponent) auditPolicyConfigMap() *corev1.ConfigMap { - const defaultAuditPolicy = `apiVersion: audit.k8s.io/v1 -kind: Policy -rules: -- level: RequestResponse - omitStages: - - RequestReceived - verbs: - - create - - patch - - update - - delete - resources: - - group: projectcalico.org - resources: - - globalnetworkpolicies - - networkpolicies - - stagedglobalnetworkpolicies - - stagednetworkpolicies - - stagedkubernetesnetworkpolicies - - globalnetworksets - - networksets - - tiers - - hostendpoints` - - return &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - // This object is for Enterprise only, so pass it explicitly. - Namespace: APIServerNamespace, - Name: auditPolicyVolumeName, - }, - Data: map[string]string{ - "config": defaultAuditPolicy, - }, - } -} - -func (c *apiServerComponent) multiTenantManagedClusterAccessClusterRoles() []client.Object { - var objects []client.Object - objects = append(objects, &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: MultiTenantManagedClustersAccessClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{ - // The Authentication Proxy in Voltron checks if Enterprise Components (using impersonation headers for - // the service in the canonical namespace) can get a managed clusters before sending the request down the tunnel. - // This ClusterRole will be assigned to each component using a RoleBinding in the canonical or tenant namespace. - "get", - }, - }, - }, - }) - - return objects -} - -// managedClusterWatchClusterRole creates a ClusterRole for watching the ManagedCluster API -func (c *apiServerComponent) managedClusterWatchClusterRole() client.Object { - return &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: ManagedClustersWatchClusterRoleName}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{ - "get", "list", "watch", - }, - }, - }, - } -} +} func (c *apiServerComponent) getDeprecatedResources() []client.Object { var renamedRscList []client.Object @@ -2492,23 +1175,21 @@ func (c *apiServerComponent) getDeprecatedResources() []client.Object { }, }) - // The following resources were not present in Calico OSS, so there is no need to clean up in OSS. - if c.cfg.Installation.Variant.IsEnterprise() { - // Renamed ClusterRoleBinging tigera-tier-getter to calico-tier-getter since Tier is available in OSS - renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-tier-getter", - }, - }) - // Renamed ClusterRole tigera-tier-getter to calico-tier-getter since Tier is available in OSS - renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-tier-getter", - }, - }) - } + // Renamed ClusterRoleBinding tigera-tier-getter to calico-tier-getter since Tier is available in OSS. + // Deleting an object that was never created (e.g. in a fresh OSS install) is a no-op. + renamedRscList = append(renamedRscList, &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-tier-getter", + }, + }) + // Renamed ClusterRole tigera-tier-getter to calico-tier-getter since Tier is available in OSS + renamedRscList = append(renamedRscList, &rbacv1.ClusterRole{ + TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-tier-getter", + }, + }) renamedRscList = append(renamedRscList, &corev1.Namespace{ TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, @@ -2532,162 +1213,3 @@ func (c *apiServerComponent) getDeprecatedResources() []client.Object { return renamedRscList } - -func (cfg *APIServerConfiguration) IsSidecarInjectionEnabled() bool { - return cfg.ApplicationLayer != nil && - cfg.ApplicationLayer.Spec.SidecarInjection != nil && - *cfg.ApplicationLayer.Spec.SidecarInjection == operatorv1.SidecarEnabled -} - -func (c *apiServerComponent) l7AdmissionControllerContainer() corev1.Container { - volumeMounts := []corev1.VolumeMount{ - c.cfg.TLSKeyPair.VolumeMount(c.SupportedOSType()), - } - - l7AdmissionControllerTargetPort := getContainerPort(c.cfg, L7AdmissionControllerContainerName).ContainerPort - - dataplane := "iptables" - if c.cfg.Installation.IsNftables() { - dataplane = "nftables" - } - - l7AdmssCtrl := corev1.Container{ - Name: string(L7AdmissionControllerContainerName), - Image: c.calicoImage, - Command: []string{components.CalicoBinaryPath, "component", "l7-admission-controller"}, - Env: []corev1.EnvVar{ - { - Name: "L7ADMCTRL_TLSCERTPATH", - Value: c.cfg.TLSKeyPair.VolumeMountCertificateFilePath(), - }, - { - Name: "L7ADMCTRL_TLSKEYPATH", - Value: c.cfg.TLSKeyPair.VolumeMountKeyFilePath(), - }, - { - Name: "L7ADMCTRL_ENVOYIMAGE", - Value: c.l7AdmissionControllerEnvoyImage, - }, - { - Name: "L7ADMCTRL_DIKASTESIMAGE", - Value: c.dikastesImage, - }, - { - Name: "L7ADMCTRL_LISTENADDR", - Value: fmt.Sprintf(":%d", l7AdmissionControllerTargetPort), - }, - { - Name: "DATAPLANE", - Value: dataplane, - }, - }, - VolumeMounts: volumeMounts, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/live", - Port: intstr.FromInt32(l7AdmissionControllerTargetPort), - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - } - - return l7AdmssCtrl -} - -// deprecatedResources removes legacy cluster-scoped resources created with the 'tigera' prefix (EE-only). -// Moving forward, both EE and OSS variants standardize on the 'calico' prefix for all shared resources. -// TODO to clean up the below deprecated logic with 14 resources in 3.25+ -func (c *apiServerComponent) deprecatedResources() []client.Object { - return []client.Object{ - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-secrets-access"}, - }, - - // delegateAuthClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-delegate-auth"}, - }, - - // authClusterRole - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, - }, - - // authClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-extension-apiserver-auth-access"}, - }, - // authReaderRoleBinding - need clean up in diff namespace kube-system - &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "tigera-auth-reader", - Namespace: "kube-system", - }, - }, - // webhookReaderClusterRole - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-webhook-reader"}, - }, - - // webhookReaderClusterRoleBinding - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver-webhook-reader"}, - }, - - // calico-apiserver CR and CRB - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-apiserver"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettingsgroup-getter"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-tiered-policy-passthrough"}, - }, - - &rbacv1.ClusterRole{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, - }, - &rbacv1.ClusterRoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-uisettings-passthrough"}, - }, - - // Clean up legacy secrets in the tigera-operator namespace - &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "tigera-api-cert", Namespace: common.OperatorNamespace()}, - }, - } -} diff --git a/pkg/render/apiserver_test.go b/pkg/render/apiserver_test.go index d4c85ac2fa..dbc65a444f 100644 --- a/pkg/render/apiserver_test.go +++ b/pkg/render/apiserver_test.go @@ -15,1580 +15,44 @@ package render_test import ( - "crypto/tls" - "crypto/x509" - "encoding/pem" "fmt" - "strings" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/onsi/gomega/gstruct" - "github.com/openshift/library-go/pkg/crypto" - - calicov3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - - operatorv1 "github.com/tigera/operator/api/v1" - "github.com/tigera/operator/pkg/apis" - "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/components" - "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/controller/k8sapi" - ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" - "github.com/tigera/operator/pkg/dns" - "github.com/tigera/operator/pkg/render" - rmeta "github.com/tigera/operator/pkg/render/common/meta" - "github.com/tigera/operator/pkg/render/common/networkpolicy" - "github.com/tigera/operator/pkg/render/common/podaffinity" - "github.com/tigera/operator/pkg/render/common/rbacmanagement" - rtest "github.com/tigera/operator/pkg/render/common/test" - "github.com/tigera/operator/pkg/render/testutils" - "github.com/tigera/operator/pkg/tls/certificatemanagement" - "github.com/tigera/operator/test" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -var _ = Describe("API server rendering tests (Calico Enterprise)", func() { - apiServerPolicy := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver.json") - apiServerPolicyForOCP := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/apiserver_ocp.json") - var ( - instance *operatorv1.InstallationSpec - apiserver *operatorv1.APIServerSpec - managementCluster = &operatorv1.ManagementCluster{Spec: operatorv1.ManagementClusterSpec{Address: "example.com:1234"}} - cfg *render.APIServerConfiguration - trustedBundle certificatemanagement.TrustedBundle - dnsNames []string - cli client.Client - certificateManager certificatemanager.CertificateManager - err error - ) - - BeforeEach(func() { - instance = &operatorv1.InstallationSpec{ - ControlPlaneReplicas: ptr.To(int32(2)), - Registry: "testregistry.com/", - Variant: operatorv1.CalicoEnterprise, - } - apiserver = &operatorv1.APIServerSpec{} - dnsNames = dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, clusterDomain) - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - - cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - certificateManager, err = certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).NotTo(HaveOccurred()) - - trustedBundle = certificatemanagement.CreateTrustedBundle(nil) - - cfg = &render.APIServerConfiguration{ - RequiresAggregationServer: true, - K8SServiceEndpoint: k8sapi.ServiceEndpoint{}, - Installation: instance, - APIServer: apiserver, - OpenShift: true, - TLSKeyPair: kp, - TrustedBundle: trustedBundle, - KubernetesVersion: &common.VersionInfo{ - Major: 1, - Minor: 31, - }, - } - }) - - DescribeTable("should render an API server with default configuration", func(clusterDomain string) { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - dnsNames := dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, clusterDomain) - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).NotTo(HaveOccurred()) - cfg.TLSKeyPair = kp - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - - resources, _ := component.Objects() - - // Should render the correct resources. - // - 1 namespace - // - 1 ConfigMap audit Policy - // - 1 ConfigMap Tigera CA bundle - // - 1 Service account - // - 2 ServiceAccount ClusterRole and binding for calico CRDs - // - 2 ServiceAccount ClusterRole and binding for tigera CRDs - // - 2 ClusterRole and binding for auth configmap - // - 2 calico policy passthru ClusterRole and binding - // - 2 tiered policy passthru ClusterRole and binding - // - 1 Role binding for tigera-operator to manage secrets - // - 1 delegate auth binding - // - 1 auth reader binding - // - 2 webhook reader ClusterRole and binding - // - 2 cert secrets - // - 1 api server - // - 1 service registration - // - 1 Server service - rtest.ExpectResources(resources, expectedResources) - - apiService, ok := rtest.GetResource(resources, "v3.projectcalico.org", "", "apiregistration.k8s.io", "v1", "APIService").(*apiregv1.APIService) - Expect(ok).To(BeTrue(), "Expected v1.APIService") - verifyAPIService(apiService, true, clusterDomain) - - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(d.Name).To(Equal("calico-apiserver")) - Expect(len(d.Labels)).To(Equal(1)) - Expect(d.Labels).To(HaveKeyWithValue("apiserver", "true")) - - Expect(*d.Spec.Replicas).To(BeEquivalentTo(2)) - Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RollingUpdateDeploymentStrategyType)) - Expect(len(d.Spec.Selector.MatchLabels)).To(Equal(1)) - Expect(d.Spec.Selector.MatchLabels).To(HaveKeyWithValue("apiserver", "true")) - - Expect(d.Spec.Template.Name).To(Equal("calico-apiserver")) - Expect(d.Spec.Template.Namespace).To(Equal("calico-system")) - Expect(len(d.Spec.Template.Labels)).To(Equal(1)) - Expect(d.Spec.Template.Labels).To(HaveKeyWithValue("apiserver", "true")) - - Expect(d.Spec.Template.Spec.ServiceAccountName).To(Equal("calico-apiserver")) - - Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateControlPlane)) - - Expect(d.Spec.Template.Spec.ImagePullSecrets).To(BeEmpty()) - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.Containers[0].Name).To(Equal("calico-apiserver")) - Expect(d.Spec.Template.Spec.Containers[0].Image).To(Equal( - fmt.Sprintf("testregistry.com/%s%s:%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, components.ComponentTigeraCalico.Version), - )) - - expectedArgs := []string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - } - Expect(d.Spec.Template.Spec.Containers[0].Args).To(ConsistOf(expectedArgs)) - Expect(len(d.Spec.Template.Spec.Containers[0].Env)).To(Equal(2)) - Expect(d.Spec.Template.Spec.Containers[0].Env[0].Name).To(Equal("DATASTORE_TYPE")) - Expect(d.Spec.Template.Spec.Containers[0].Env[0].Value).To(Equal("kubernetes")) - Expect(d.Spec.Template.Spec.Containers[0].Env[0].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[0].Env[1].Name).To(Equal("LOG_LEVEL")) - Expect(d.Spec.Template.Spec.Containers[0].Env[1].Value).To(Equal("info")) - Expect(d.Spec.Template.Spec.Containers[0].Env[1].ValueFrom).To(BeNil()) - - Expect(len(d.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(3)) - Expect(d.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath).To(Equal("/var/log/calico/audit")) - Expect(d.Spec.Template.Spec.Containers[0].VolumeMounts[1].Name).To(Equal("calico-audit-logs")) - - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Path).To(Equal("/readyz")) - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Port.String()).To(BeEquivalentTo("5443")) - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Scheme).To(BeEquivalentTo("HTTPS")) - Expect(d.Spec.Template.Spec.Containers[0].ReadinessProbe.PeriodSeconds).To(BeEquivalentTo(60)) - - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeTrue()) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeTrue()) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(0)) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.RunAsNonRoot).To(BeFalse()) - Expect(*d.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser).To(BeEquivalentTo(0)) - Expect(d.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities).To(Equal( - &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - )) - Expect(d.Spec.Template.Spec.Containers[0].SecurityContext.SeccompProfile).To(Equal( - &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - })) - - Expect(d.Spec.Template.Spec.Containers[1].Name).To(Equal("tigera-queryserver")) - Expect(d.Spec.Template.Spec.Containers[1].Image).To(Equal( - fmt.Sprintf("testregistry.com/%s%s:%s", components.TigeraImagePath, components.ComponentTigeraCalico.Image, components.ComponentTigeraCalico.Version), - )) - Expect(d.Spec.Template.Spec.Containers[1].Args).To(BeEmpty()) - - Expect(d.Spec.Template.Spec.Containers[1].Env).To(HaveLen(10)) - - Expect(d.Spec.Template.Spec.Containers[1].Env[0].Name).To(Equal("DATASTORE_TYPE")) - Expect(d.Spec.Template.Spec.Containers[1].Env[0].Value).To(Equal("kubernetes")) - Expect(d.Spec.Template.Spec.Containers[1].Env[0].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[1].Name).To(Equal("LISTEN_ADDR")) - Expect(d.Spec.Template.Spec.Containers[1].Env[1].Value).To(Equal(":8080")) - Expect(d.Spec.Template.Spec.Containers[1].Env[1].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[2].Name).To(Equal("TLS_CERT")) - Expect(d.Spec.Template.Spec.Containers[1].Env[2].Value).To(Equal("/calico-apiserver-certs/tls.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[2].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[3].Name).To(Equal("TLS_KEY")) - Expect(d.Spec.Template.Spec.Containers[1].Env[3].Value).To(Equal("/calico-apiserver-certs/tls.key")) - Expect(d.Spec.Template.Spec.Containers[1].Env[3].ValueFrom).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].Env[4].Name).To(Equal("TRUSTED_BUNDLE_PATH")) - Expect(d.Spec.Template.Spec.Containers[1].Env[4].Value).To(Equal("/etc/pki/tls/certs/tigera-ca-bundle.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[5].Name).To(Equal("LINSEED_URL")) - Expect(d.Spec.Template.Spec.Containers[1].Env[5].Value).To(Equal("https://tigera-linseed.tigera-elasticsearch.svc")) - Expect(d.Spec.Template.Spec.Containers[1].Env[6].Name).To(Equal("LINSEED_CLIENT_CERT")) - Expect(d.Spec.Template.Spec.Containers[1].Env[6].Value).To(Equal("/calico-apiserver-certs/tls.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[7].Name).To(Equal("LINSEED_CLIENT_KEY")) - Expect(d.Spec.Template.Spec.Containers[1].Env[7].Value).To(Equal("/calico-apiserver-certs/tls.key")) - Expect(d.Spec.Template.Spec.Containers[1].Env[8].Name).To(Equal("LINSEED_CA")) - Expect(d.Spec.Template.Spec.Containers[1].Env[8].Value).To(Equal("/etc/pki/tls/certs/tigera-ca-bundle.crt")) - Expect(d.Spec.Template.Spec.Containers[1].Env[9].Name).To(Equal("LOGLEVEL")) - Expect(d.Spec.Template.Spec.Containers[1].Env[9].Value).To(Equal("info")) - Expect(d.Spec.Template.Spec.Containers[1].Env[9].ValueFrom).To(BeNil()) - - // Expect the SECURITY_GROUP env variables to not be set - Expect(d.Spec.Template.Spec.Containers[1].Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(d.Spec.Template.Spec.Containers[1].Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].Name).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].MountPath).To(Equal("/calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].ReadOnly).To(BeTrue()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].SubPath).To(Equal("")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].MountPropagation).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[0].SubPathExpr).To(Equal("")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].Name).To(Equal("tigera-ca-bundle")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPath).To(Equal("/etc/pki/tls/certs")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].ReadOnly).To(BeTrue()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].SubPath).To(Equal("")) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].MountPropagation).To(BeNil()) - Expect(d.Spec.Template.Spec.Containers[1].VolumeMounts[1].SubPathExpr).To(Equal("")) - - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.HTTPGet.Path).To(Equal("/version")) - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.HTTPGet.Port.String()).To(BeEquivalentTo("8080")) - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.HTTPGet.Scheme).To(BeEquivalentTo("HTTPS")) - Expect(d.Spec.Template.Spec.Containers[1].LivenessProbe.InitialDelaySeconds).To(BeEquivalentTo(90)) - - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.Privileged).To(BeFalse()) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.RunAsGroup).To(BeEquivalentTo(10001)) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.RunAsNonRoot).To(BeTrue()) - Expect(*d.Spec.Template.Spec.Containers[1].SecurityContext.RunAsUser).To(BeEquivalentTo(10001)) - Expect(d.Spec.Template.Spec.Containers[1].SecurityContext.Capabilities).To(Equal( - &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - )) - Expect(d.Spec.Template.Spec.Containers[1].SecurityContext.SeccompProfile).To(Equal( - &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - })) - - Expect(d.Spec.Template.Spec.Volumes).To(HaveLen(4)) - Expect(d.Spec.Template.Spec.Volumes[0].Name).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Volumes[0].Secret.SecretName).To(Equal("calico-apiserver-certs")) - Expect(d.Spec.Template.Spec.Volumes[1].Name).To(Equal("calico-audit-logs")) - Expect(d.Spec.Template.Spec.Volumes[1].HostPath.Path).To(Equal("/var/log/calico/audit")) - Expect(*d.Spec.Template.Spec.Volumes[1].HostPath.Type).To(BeEquivalentTo("DirectoryOrCreate")) - Expect(d.Spec.Template.Spec.Volumes[2].Name).To(Equal("calico-audit-policy")) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Name).To(Equal("calico-audit-policy")) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Items).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Items[0].Key).To(Equal("config")) - Expect(d.Spec.Template.Spec.Volumes[2].ConfigMap.Items[0].Path).To(Equal("policy.conf")) - Expect(d.Spec.Template.Spec.Volumes[3].Name).To(Equal("tigera-ca-bundle")) - Expect(d.Spec.Template.Spec.Volumes[3].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - - clusterRole := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ConsistOf(networkAdminPolicyRules)) - - clusterRole = rtest.GetResource(resources, "tigera-ui-user", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ConsistOf(uiUserPolicyRules)) - - clusterRoleBinding := rtest.GetResource(resources, "calico-extension-apiserver-auth-access", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(clusterRoleBinding.RoleRef.Name).To(Equal("calico-extension-apiserver-auth-access")) - - svc := rtest.GetResource(resources, "calico-api", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(svc.GetObjectMeta().GetLabels()).To(HaveLen(1)) - Expect(svc.GetObjectMeta().GetLabels()).To(HaveKeyWithValue("k8s-app", "calico-api")) - - Expect(svc.Spec.Ports).To(HaveLen(2)) - serviceFound := 0 - for _, p := range svc.Spec.Ports { - switch p.Name { - case render.APIServerPortName: - Expect(p.Port).To(Equal(int32(443))) - Expect(p.TargetPort.IntValue()).To(Equal(5443)) - serviceFound++ - case render.QueryServerPortName: - Expect(p.Port).To(Equal(int32(8080))) - Expect(p.TargetPort.IntValue()).To(Equal(8080)) - serviceFound++ - } - } - Expect(serviceFound).To(Equal(2)) - - cr := rtest.GetResource(resources, "calico-tiered-policy-passthrough", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - var tieredPolicyRules []string - for _, rule := range cr.Rules { - tieredPolicyRules = append(tieredPolicyRules, rule.Resources...) - } - Expect(tieredPolicyRules).To(ContainElements("networkpolicies", "globalnetworkpolicies", "stagednetworkpolicies", "stagedglobalnetworkpolicies")) - - apiserverClusterRole := rtest.GetResource(resources, - "calico-crds", "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(apiserverClusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"admissionregistration.k8s.io"}, - Resources: []string{ - "validatingadmissionpolicies", - "validatingadmissionpolicybindings", - }, - Verbs: []string{ - "get", - "list", - "watch", - }, - })) - }, - Entry("default cluster domain", dns.DefaultClusterDomain), - Entry("custom cluster domain", "custom-domain.internal"), - ) - - // The escalation-capable rolebinding rules are the reason this is gated. - It("should gate the RBAC management UI rules on tigera-network-admin", func() { - By("omitting them while the feature gate is off") - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - resources, _ := component.Objects() - clusterRole := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - for _, rule := range rbacManagementNetworkAdminRules { - Expect(clusterRole.Rules).NotTo(ContainElement(rule)) - } - Expect(clusterRole.Rules).To(ConsistOf(networkAdminPolicyRules)) - - By("adding them once the admin switches the feature on") - cfg.RBACManagementEnabled = true - component, err = render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - resources, _ = component.Objects() - clusterRole = rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ConsistOf(append(networkAdminPolicyRules, rbacManagementNetworkAdminRules...))) - }) - - // Not gated: a rule rendered only while the feature is on could never turn it on. - It("should grant tigera-network-admin write access to the switch regardless of the gate", func() { - gateWriteRules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{rbacmanagement.ConfigMapName}, - Verbs: []string{"get", "list", "watch", "update", "patch", "delete"}, - }, - } - - for _, enabled := range []bool{false, true} { - cfg.RBACManagementEnabled = enabled - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - resources, _ := component.Objects() - clusterRole := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - for _, rule := range gateWriteRules { - Expect(clusterRole.Rules).To(ContainElement(rule), - "expected the switch write rules with RBACManagementEnabled=%v", enabled) - } - } - }) - - It("should render resources without an aggregation server", func() { - cfg.RequiresAggregationServer = false - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - rtest.ExpectResources(resources, expectedResources) - - // In CRD/webhooks mode the calico-uisettings-passthrough ClusterRole is not deployed, so - // tigera-network-admin needs to grant write access to uisettings itself for the - // calico-webhooks UISettings handler to do the narrowing instead of being short-circuited - // by kube-apiserver RBAC. - networkAdmin := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(networkAdmin.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings"}, - Verbs: []string{"create", "update", "delete", "patch"}, - })) - }) - - It("should keep the apiserver deployment but drop the APIService for enterprise in v3 CRD mode", func() { - cfg.RequiresAggregationServer = false - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - // The Deployment must still be present: in enterprise it hosts the queryserver. - Expect(rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment")).ToNot(BeNil()) - - // The v3.projectcalico.org APIService must be absent in v3 CRD mode. - for _, r := range resources { - Expect(r.GetObjectKind().GroupVersionKind().Kind).NotTo(Equal("APIService"), - "unexpected APIService registered in v3 CRD mode: %s", r.GetName()) - } - }) - - It("binds the previous API server to the CRD role while the cutover is held", func() { - // v1.38 created calico-apiserver-access-calico-crds with the previous service account, so - // narrowing it to one subject mid-migration stops the API server that is still serving. - subjectsFor := func(hold bool) []rbacv1.Subject { - cfg.HoldAPIServiceCutover = hold - component, err := render.APIServer(cfg) - Expect(err).To(BeNil()) - resources, _ := component.Objects() - crb := rtest.GetResource(resources, "calico-apiserver-access-calico-crds", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding") - Expect(crb).NotTo(BeNil()) - return crb.(*rbacv1.ClusterRoleBinding).Subjects - } - - held := subjectsFor(true) - Expect(held).To(ContainElement(rbacv1.Subject{Kind: "ServiceAccount", Name: "calico-apiserver", Namespace: "calico-system"})) - Expect(held).To(ContainElement(rbacv1.Subject{Kind: "ServiceAccount", Name: "tigera-apiserver", Namespace: "tigera-system"})) - - done := subjectsFor(false) - Expect(done).To(HaveLen(1)) - Expect(done).To(ContainElement(rbacv1.Subject{Kind: "ServiceAccount", Name: "calico-apiserver", Namespace: "calico-system"})) - }) - - It("should grant the calico-apiserver SA write access to globalreports/status", func() { - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - cr := rtest.GetResource(resources, "calico-apiserver", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - - // Find the backing-storage rule and assert it covers globalreports/status. - // Without this, compliance-controller can't update status.lastScheduledReportJob - // and no compliance jobs ever run. - var found bool - for _, rule := range cr.Rules { - hasGlobalReports := false - hasGlobalReportsStatus := false - for _, r := range rule.Resources { - if r == "globalreports" { - hasGlobalReports = true - } - if r == "globalreports/status" { - hasGlobalReportsStatus = true - } - } - if hasGlobalReports { - found = true - Expect(hasGlobalReportsStatus).To(BeTrue(), "calico-apiserver ClusterRole rule covering globalreports must also cover globalreports/status") - Expect(rule.Verbs).To(ContainElement("update")) - } - } - Expect(found).To(BeTrue(), "calico-apiserver ClusterRole should have a rule covering globalreports") - }) - - It("should render L7 Admission Controller with default config when SidecarInjection is Enabled", func() { - sidecarEnabled := operatorv1.SidecarEnabled - cfg.ApplicationLayer = &operatorv1.ApplicationLayer{ - Spec: operatorv1.ApplicationLayerSpec{ - SidecarInjection: &sidecarEnabled, - }, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - var container corev1.Container - for _, c := range d.Spec.Template.Spec.Containers { - if c.Name == "calico-l7-admission-controller" { - container = c - } - } - Expect(container.Env[4].Name).To(Equal("L7ADMCTRL_LISTENADDR")) - Expect(container.Env[4].Value).To(Equal(":6443")) - - // Check the Service configuration - svc := rtest.GetResource(resources, "calico-api", "calico-system", "", "v1", "Service").(*corev1.Service) - var servicePort corev1.ServicePort - for _, p := range svc.Spec.Ports { - if p.Name == render.L7AdmissionControllerPortName { - servicePort = p - } - } - Expect(servicePort.Port).To(Equal(int32(6443))) - Expect(servicePort.TargetPort.IntValue()).To(Equal(6443)) - }) - - It("should render log severity when provided", func() { - errorLog := operatorv1.LogSeverityError - debugLog := operatorv1.LogSeverityDebug - cfg.APIServer.Logging = &operatorv1.APIServerPodLogging{ - APIServerLogging: &operatorv1.APIServerLogging{ - LogSeverity: &errorLog, - }, - QueryServerLogging: &operatorv1.QueryServerLogging{ - LogSeverity: &debugLog, - }, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - - containers := deploy.Spec.Template.Spec.Containers - for _, container := range containers { - envs := container.Env - if strings.Contains(container.Name, "apiserver") { - for _, env := range envs { - if env.Name == "LOG_LEVEL" { - Expect(env.Value).To(Equal("error")) - } - } - } else if strings.Contains(container.Name, "queryserver") { - for _, env := range envs { - if env.Name == "LOGLEVEL" { - Expect(env.Value).To(Equal("debug")) - } - } - } - } - Expect(deploy.Spec.Template.Spec.Containers).NotTo(BeNil()) - Expect(deploy.Spec.Template.Spec.Affinity).To(Equal(podaffinity.NewPodAntiAffinity("calico-apiserver", []string{"calico-system", "tigera-system", "calico-apiserver"}))) - }) - - It("should render SecurityContextConstrains properly when provider is OpenShift", func() { - cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.Installation.Variant = operatorv1.CalicoEnterprise - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - role := rtest.GetResource(resources, "calico-extension-apiserver-auth-access", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"privileged"}, - })) - Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"config.openshift.io"}, - Resources: []string{"infrastructures"}, - Verbs: []string{"get", "list", "watch"}, - })) - }) - - It("should render an API server with custom configuration", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - rtest.ExpectResources(resources, expectedResources) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - rtest.ExpectResourceTypeAndObjectMetadata(dep, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - d := dep.(*appsv1.Deployment) - - Expect(d.Spec.Template.Spec.Volumes).To(HaveLen(4)) - }) - - It("should render needed resources for k8s kube-controller", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - rtest.ExpectResources(resources, expectedResources) - - // Should render the correct resources. - cr := rtest.GetResource(resources, "calico-tier-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(len(cr.Rules)).To(Equal(1)) - Expect(len(cr.Rules[0].Resources)).To(Equal(1)) - Expect(cr.Rules[0].Resources[0]).To(Equal("tiers")) - Expect(len(cr.Rules[0].Verbs)).To(Equal(1)) - Expect(cr.Rules[0].Verbs[0]).To(Equal("get")) - - crb := rtest.GetResource(resources, "calico-tier-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Kind).To(Equal("ClusterRole")) - Expect(crb.RoleRef.Name).To(Equal("calico-tier-getter")) - Expect(len(crb.Subjects)).To(Equal(1)) - Expect(crb.Subjects[0].Kind).To(Equal("User")) - Expect(crb.Subjects[0].Name).To(Equal("system:kube-controller-manager")) - - cr = rtest.GetResource(resources, "calico-uisettingsgroup-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(len(cr.Rules)).To(Equal(1)) - Expect(len(cr.Rules[0].Resources)).To(Equal(1)) - Expect(cr.Rules[0].Resources[0]).To(Equal("uisettingsgroups")) - Expect(len(cr.Rules[0].Verbs)).To(Equal(1)) - Expect(cr.Rules[0].Verbs[0]).To(Equal("get")) - - crb = rtest.GetResource(resources, "calico-uisettingsgroup-getter", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Kind).To(Equal("ClusterRole")) - Expect(crb.RoleRef.Name).To(Equal("calico-uisettingsgroup-getter")) - Expect(len(crb.Subjects)).To(Equal(1)) - Expect(crb.Subjects[0].Kind).To(Equal("User")) - Expect(crb.Subjects[0].Name).To(Equal("system:kube-controller-manager")) - }) - - It("should include a ControlPlaneNodeSelector when specified", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - cfg.Installation.ControlPlaneNodeSelector = map[string]string{"nodeName": "control01"} - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - rtest.ExpectResources(resources, expectedResources) - - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) - }) - - It("should include a ControlPlaneToleration when specified", func() { - tol := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - Effect: corev1.TaintEffectNoExecute, - } - cfg.Installation.ControlPlaneTolerations = []corev1.Toleration{tol} - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) - }) - - It("should include a ClusterRole and ClusterRoleBindings for reading webhook configuration", func() { - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - rtest.ExpectResources(resources, expectedResources) - - // Should render the correct resources. - cr := rtest.GetResource(resources, "calico-webhook-reader", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(len(cr.Rules)).To(Equal(1)) - Expect(len(cr.Rules[0].Resources)).To(Equal(2)) - Expect(cr.Rules[0].Resources[0]).To(Equal("mutatingwebhookconfigurations")) - Expect(cr.Rules[0].Resources[1]).To(Equal("validatingwebhookconfigurations")) - Expect(len(cr.Rules[0].Verbs)).To(Equal(3)) - Expect(cr.Rules[0].Verbs[0]).To(Equal("get")) - Expect(cr.Rules[0].Verbs[1]).To(Equal("list")) - Expect(cr.Rules[0].Verbs[2]).To(Equal("watch")) - - crb := rtest.GetResource(resources, "calico-apiserver-webhook-reader", "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Kind).To(Equal("ClusterRole")) - Expect(crb.RoleRef.Name).To(Equal("calico-webhook-reader")) - Expect(len(crb.Subjects)).To(Equal(1)) - Expect(crb.Subjects[0].Kind).To(Equal("ServiceAccount")) - Expect(crb.Subjects[0].Name).To(Equal("calico-apiserver")) - Expect(crb.Subjects[0].Namespace).To(Equal("calico-system")) - }) - - It("should set KUBERENETES_SERVICE_... variables if host networked", func() { - cfg.K8SServiceEndpoint.Host = "k8shost" - cfg.K8SServiceEndpoint.Port = "1234" - cfg.ForceHostNetwork = true - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(deploymentResource).ToNot(BeNil()) - - deployment := deploymentResource.(*appsv1.Deployment) - rtest.ExpectK8sServiceEpEnvVars(deployment.Spec.Template.Spec, "k8shost", "1234") - }) - - It("should set RecreateDeploymentStrategyType if host networked", func() { - cfg.ForceHostNetwork = true - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) - }) - - It("should add egress policy with Enterprise variant and K8SServiceEndpoint defined", func() { - cfg.K8SServiceEndpoint.Host = "k8shost" - cfg.K8SServiceEndpoint.Port = "1234" - cfg.ForceHostNetwork = true - - component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() - policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).ToNot(BeNil()) - Expect(policy.Spec).ToNot(BeNil()) - Expect(policy.Spec.Egress).ToNot(BeNil()) - Expect(policy.Spec.Egress).To(ContainElement(calicov3.Rule{ - Action: calicov3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: calicov3.EntityRule{ - Ports: networkpolicy.Ports(1234), - Domains: []string{"k8shost"}, - }, - })) - }) - - It("should add egress policy with Enterprise variant and K8SServiceEndpoint as IP defined", func() { - cfg.K8SServiceEndpoint.Host = "169.169.169.169" - cfg.K8SServiceEndpoint.Port = "4321" - cfg.ForceHostNetwork = false - - component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() - policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).ToNot(BeNil()) - Expect(policy.Spec).ToNot(BeNil()) - Expect(policy.Spec.Egress).ToNot(BeNil()) - Expect(policy.Spec.Egress).To(ContainElement(calicov3.Rule{ - Action: calicov3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: calicov3.EntityRule{ - Ports: networkpolicy.Ports(4321), - Nets: []string{"169.169.169.169/32"}, - }, - })) - }) - - It("should not set KUBERENETES_SERVICE_... variables if not host networked on Docker EE with proxy.local", func() { - cfg.K8SServiceEndpoint.Host = "proxy.local" - cfg.K8SServiceEndpoint.Port = "1234" - cfg.Installation.KubernetesProvider = operatorv1.ProviderDockerEE - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(deploymentResource).ToNot(BeNil()) - - deployment := deploymentResource.(*appsv1.Deployment) - rtest.ExpectNoK8sServiceEpEnvVars(deployment.Spec.Template.Spec) - }) - - It("should set KUBERENETES_SERVICE_... variables if not host networked on Docker EE with non-proxy address", func() { - cfg.K8SServiceEndpointPodNetwork.Host = "k8shost" - cfg.K8SServiceEndpointPodNetwork.Port = "1234" - cfg.Installation.KubernetesProvider = operatorv1.ProviderDockerEE - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(deploymentResource).ToNot(BeNil()) - - deployment := deploymentResource.(*appsv1.Deployment) - rtest.ExpectK8sServiceEpEnvVars(deployment.Spec.Template.Spec, "k8shost", "1234") - }) - - It("should render an API server with custom configuration with MCM enabled at startup", func() { - cfg.ManagementCluster = managementCluster - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - - resources, _ := component.Objects() - - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - - rtest.ExpectResources(resources, expectedResources) - - By("Validating the newly created tunnel secret") - tunnelSecret, err := certificatemanagement.CreateSelfSignedSecret(render.VoltronTunnelSecretName, common.OperatorNamespace(), "tigera-voltron", []string{"voltron"}) - Expect(err).ToNot(HaveOccurred()) - - // Use the x509 package to validate that the cert was signed with the privatekey - validateTunnelSecret(tunnelSecret) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - By("Validating startup args") - expectedArgs := []string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - "--enable-managed-clusters-create-api=true", - "--managementClusterAddr=example.com:1234", - } - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ConsistOf(expectedArgs)) - }) - - It("should render an API server with custom configuration with MCM enabled at restart", func() { - cfg.ManagementCluster = managementCluster - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - - resources, _ := component.Objects() - - expected := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.ManagedClustersWatchClusterRoleName}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}}, - &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.APIServerSecretsRBACName, Namespace: "calico-system"}}, - } - rtest.ExpectResources(resources, expected) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - By("Validating startup args") - expectedArgs := []string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - "--enable-managed-clusters-create-api=true", - "--managementClusterAddr=example.com:1234", - } - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ConsistOf(expectedArgs)) - }) - - It("should render an API server with signed ca bundles enabled", func() { - cfg.ManagementCluster = managementCluster - cfg.ManagementCluster.Spec.TLS = &operatorv1.TLS{ - SecretName: render.ManagerTLSSecretName, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - - resources, _ := component.Objects() - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ContainElement("--managementClusterCAType=Public")) - Expect((dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args).To(ContainElement(fmt.Sprintf("--tunnelSecretName=%s", render.ManagerTLSSecretName))) - }) - - It("should pass tunnelSecretName when TLS secret is not manager-tls", func() { - cfg.ManagementCluster = managementCluster - cfg.ManagementCluster.Spec.TLS = &operatorv1.TLS{ - SecretName: render.VoltronTunnelSecretName, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - - resources, _ := component.Objects() - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - - args := (dep.(*appsv1.Deployment)).Spec.Template.Spec.Containers[0].Args - Expect(args).To(ContainElement(fmt.Sprintf("--tunnelSecretName=%s", render.VoltronTunnelSecretName))) - Expect(args).ToNot(ContainElement("--managementClusterCAType=Public")) - }) - - It("should add an init container if certificate management is enabled", func() { - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{SignerName: "a.b/c", CACert: cfg.TLSKeyPair.GetCertificatePEM()} - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).NotTo(HaveOccurred()) - qskp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - cfg.TLSKeyPair = kp - cfg.QueryServerTLSKeyPairCertificateManagementOnly = qskp - Expect(err).NotTo(HaveOccurred()) - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - expectedResources := []client.Object{ - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "calico-audit-policy", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ca-bundle", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-access-calico-crds"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tier-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-tiered-policy-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettings-passthrough"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-extension-apiserver-auth-access"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-delegate-auth"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-auth-reader", Namespace: "kube-system"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &apiregv1.APIService{ObjectMeta: metav1.ObjectMeta{Name: "v3.projectcalico.org"}, TypeMeta: metav1.TypeMeta{Kind: "APIService", APIVersion: "apiregistration.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-api", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "PodDisruptionBudget", APIVersion: "policy/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-uisettingsgroup-getter"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-ui-user"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-network-admin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-apiserver-webhook-reader"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - } - rtest.ExpectResources(resources, expectedResources) - - dep := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") - Expect(dep).ToNot(BeNil()) - deploy, ok := dep.(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) - Expect(deploy.Spec.Template.Spec.InitContainers[0].Name).To(Equal("calico-apiserver-certs-key-cert-provisioner")) - rtest.ExpectEnv(deploy.Spec.Template.Spec.InitContainers[0].Env, "SIGNER", "a.b/c") - }) - - It("should not render PodAffinity when ControlPlaneReplicas is 1", func() { - cfg.Installation.ControlPlaneReplicas = ptr.To(int32(1)) - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Affinity).To(BeNil()) - }) - - It("should render PodAffinity when ControlPlaneReplicas is greater than 1", func() { - cfg.Installation.ControlPlaneReplicas = ptr.To(int32(2)) - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Affinity).NotTo(BeNil()) - Expect(deploy.Spec.Template.Spec.Affinity).To(Equal(podaffinity.NewPodAntiAffinity("calico-apiserver", []string{"calico-system", "tigera-system", "calico-apiserver"}))) - }) - - It("should render Linseed routing for the queryserver when ManagementClusterConnection is set", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} - cfg.ClusterDomain = "cluster.local" - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - rb, ok := rtest.GetResource(resources, "tigera-linseed", "calico-system", "rbac.authorization.k8s.io", "v1", "RoleBinding").(*rbacv1.RoleBinding) - Expect(ok).To(BeTrue(), "expected tigera-linseed RoleBinding in calico-system") - Expect(rb.RoleRef.Name).To(Equal("tigera-linseed-secrets")) - Expect(rb.Subjects).To(ConsistOf(rbacv1.Subject{ - Kind: "ServiceAccount", - Name: render.GuardianServiceAccountName, - Namespace: render.GuardianNamespace, - })) - - deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - var qs *corev1.Container - for i := range deploy.Spec.Template.Spec.Containers { - if deploy.Spec.Template.Spec.Containers[i].Name == "tigera-queryserver" { - qs = &deploy.Spec.Template.Spec.Containers[i] - } - } - Expect(qs).NotTo(BeNil()) - Expect(qs.Env).To(ContainElement(corev1.EnvVar{Name: "LINSEED_URL", Value: "https://guardian.calico-system.svc"})) - Expect(qs.Env).To(ContainElement(corev1.EnvVar{Name: "CLUSTER_ID", Value: ""})) - Expect(qs.Env).To(ContainElement(corev1.EnvVar{Name: "LINSEED_TOKEN", Value: "/var/run/secrets/tigera.io/linseed/token"})) - Expect(qs.VolumeMounts).To(ContainElement(corev1.VolumeMount{ - Name: render.LinseedTokenVolumeName, - MountPath: render.LinseedVolumeMountPath, - })) - - var tokenVol *corev1.Volume - for i := range deploy.Spec.Template.Spec.Volumes { - if deploy.Spec.Template.Spec.Volumes[i].Name == render.LinseedTokenVolumeName { - tokenVol = &deploy.Spec.Template.Spec.Volumes[i] - } - } - Expect(tokenVol).NotTo(BeNil()) - Expect(tokenVol.Secret).NotTo(BeNil()) - Expect(tokenVol.Secret.SecretName).To(Equal("calico-apiserver-tigera-linseed-token")) - }) - - Context("calico-system rendering", func() { - policyName := types.NamespacedName{Name: "calico-system.apiserver-access", Namespace: "calico-system"} - - DescribeTable("should render calico-system policy", - func(scenario testutils.CalicoSystemScenario) { - cfg.OpenShift = scenario.OpenShift - if scenario.ManagedCluster { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} - } else { - cfg.ManagementClusterConnection = nil - } - - component := render.APIServerPolicy(cfg) - resources, _ := component.Objects() - - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - expectedPolicy := testutils.SelectPolicyByProvider(scenario, apiServerPolicy, apiServerPolicyForOCP) - Expect(policy).To(Equal(expectedPolicy)) - }, - Entry("for management/standalone, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: false}), - Entry("for management/standalone, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: true}), - Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), - Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), - ) - }) - - Context("With APIServer Deployment overrides", func() { - rr1 := corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("2"), - "memory": resource.MustParse("300Mi"), - "storage": resource.MustParse("20Gi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("1"), - "memory": resource.MustParse("150Mi"), - "storage": resource.MustParse("10Gi"), - }, - } - - rr2 := corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("250m"), - corev1.ResourceMemory: resource.MustParse("64Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("500m"), - corev1.ResourceMemory: resource.MustParse("500Mi"), - }, - } - - It("should handle APIServerDeployment overrides", func() { - var minReadySeconds int32 = 20 - - affinity := &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{{ - MatchExpressions: []corev1.NodeSelectorRequirement{{ - Key: "custom-affinity-key", - Operator: corev1.NodeSelectorOpExists, - }}, - }}, - }, - }, - } - toleration := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - } - - apiServerPort := operatorv1.APIServerDeploymentContainerPort{ - Name: render.APIServerPortName, - ContainerPort: 1111, - } - queryServerPort := operatorv1.APIServerDeploymentContainerPort{ - Name: render.QueryServerPortName, - ContainerPort: 2222, - } - l7AdmCtrlPort := operatorv1.APIServerDeploymentContainerPort{ - Name: render.L7AdmissionControllerPortName, - ContainerPort: 3333, - } - - sidecarEnabled := operatorv1.SidecarEnabled - cfg.ApplicationLayer = &operatorv1.ApplicationLayer{ - Spec: operatorv1.ApplicationLayerSpec{ - SidecarInjection: &sidecarEnabled, - }, - } - - cfg.APIServer.APIServerDeployment = &operatorv1.APIServerDeployment{ - Metadata: &operatorv1.Metadata{ - Labels: map[string]string{"top-level": "label1"}, - Annotations: map[string]string{"top-level": "annot1"}, - }, - Spec: &operatorv1.APIServerDeploymentSpec{ - MinReadySeconds: &minReadySeconds, - Template: &operatorv1.APIServerDeploymentPodTemplateSpec{ - Metadata: &operatorv1.Metadata{ - Labels: map[string]string{"template-level": "label2"}, - Annotations: map[string]string{"template-level": "annot2"}, - }, - Spec: &operatorv1.APIServerDeploymentPodSpec{ - Containers: []operatorv1.APIServerDeploymentContainer{ - { - Name: "calico-apiserver", - Resources: &rr1, - Ports: []operatorv1.APIServerDeploymentContainerPort{apiServerPort}, - }, - { - Name: "tigera-queryserver", - Resources: &rr2, - Ports: []operatorv1.APIServerDeploymentContainerPort{queryServerPort}, - }, - { - Name: "calico-l7-admission-controller", - Resources: &rr2, - Ports: []operatorv1.APIServerDeploymentContainerPort{l7AdmCtrlPort}, - }, - }, - InitContainers: []operatorv1.APIServerDeploymentInitContainer{ - { - Name: "calico-apiserver-certs-key-cert-provisioner", - Resources: &rr2, - }, - }, - NodeSelector: map[string]string{ - "custom-node-selector": "value", - }, - TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ - { - MaxSkew: 1, - }, - }, - Affinity: affinity, - Tolerations: []corev1.Toleration{toleration}, - }, - }, - }, - } - // Enable certificate management. - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{SignerName: "a.b/c", CACert: cfg.TLSKeyPair.GetCertificatePEM()} - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - // Create and add the TLS keypair so the initContainer is rendered. - dnsNames := dns.GetServiceDNSNames(render.APIServerServiceName, render.APIServerNamespace, clusterDomain) - kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - cfg.TLSKeyPair = kp - - qsKP, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - cfg.QueryServerTLSKeyPairCertificateManagementOnly = qsKP - - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - - // API server has apiserver: true label - Expect(d.Labels).To(HaveLen(2)) - Expect(d.Labels["apiserver"]).To(Equal("true")) - Expect(d.Labels["top-level"]).To(Equal("label1")) - Expect(d.Annotations).To(HaveLen(2)) - Expect(d.Annotations["top-level"]).To(Equal("annot1")) - // The render package records the applied resource override in an annotation. - Expect(d.Annotations["operator.tigera.io/custom-overrides"]).To(Equal("resources")) - - Expect(d.Spec.MinReadySeconds).To(Equal(minReadySeconds)) - - // At runtime, the operator will also add some standard labels to the - // deployment such as "k8s-app=calico-apiserver". But the APIServer - // deployment object produced by the render will have no labels so we expect just the one - // provided. - Expect(d.Spec.Template.Labels).To(HaveLen(2)) - Expect(d.Spec.Template.Labels["apiserver"]).To(Equal("true")) - Expect(d.Spec.Template.Labels["template-level"]).To(Equal("label2")) - - // With the default instance we expect 2 template-level annotations - // - 1 added by the operator by default - // - 1 added by the calicoNodeDaemonSet override - Expect(d.Spec.Template.Annotations).To(HaveLen(2)) - Expect(d.Spec.Template.Annotations).To(HaveKey("tigera-operator.hash.operator.tigera.io/calico-apiserver-certs")) - Expect(d.Spec.Template.Annotations["template-level"]).To(Equal("annot2")) - - Expect(d.Spec.Template.Spec.Containers).To(HaveLen(3)) - containersFound := 0 - for _, c := range d.Spec.Template.Spec.Containers { - switch c.Name { - case "calico-apiserver": - Expect(c.Resources).To(Equal(rr1)) - Expect(c.Ports[0].Name).To(Equal(apiServerPort.Name)) - Expect(c.Ports[0].ContainerPort).To(Equal(apiServerPort.ContainerPort)) - - Expect(c.Args[0]).To(ContainSubstring(fmt.Sprintf("--secure-port=%d", apiServerPort.ContainerPort))) - containersFound++ - case "tigera-queryserver": - Expect(c.Resources).To(Equal(rr2)) - Expect(c.Ports[0].Name).To(Equal(queryServerPort.Name)) - Expect(c.Ports[0].ContainerPort).To(Equal(queryServerPort.ContainerPort)) - - Expect(c.Env[1].Name).To(Equal("LISTEN_ADDR")) - Expect(c.Env[1].Value).To(Equal(fmt.Sprintf(":%d", queryServerPort.ContainerPort))) - containersFound++ - case "calico-l7-admission-controller": - Expect(c.Resources).To(Equal(rr2)) - Expect(c.Ports[0].Name).To(Equal(l7AdmCtrlPort.Name)) - Expect(c.Ports[0].ContainerPort).To(Equal(l7AdmCtrlPort.ContainerPort)) - - Expect(c.Env[4].Name).To(Equal("L7ADMCTRL_LISTENADDR")) - Expect(c.Env[4].Value).To(Equal(fmt.Sprintf(":%d", l7AdmCtrlPort.ContainerPort))) - containersFound++ - } - } - Expect(containersFound).To(Equal(3)) - - Expect(d.Spec.Template.Spec.InitContainers).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.InitContainers[0].Name).To(Equal("calico-apiserver-certs-key-cert-provisioner")) - Expect(d.Spec.Template.Spec.InitContainers[0].Resources).To(Equal(rr2)) - - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("custom-node-selector", "value")) - - Expect(d.Spec.Template.Spec.TopologySpreadConstraints).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.TopologySpreadConstraints[0].MaxSkew).To(Equal(int32(1))) - - Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.Tolerations[0]).To(Equal(toleration)) - - // Check the Service configuration - svc := rtest.GetResource(resources, "calico-api", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(svc.Spec.Ports).To(HaveLen(3)) - servicesFound := 0 - for _, p := range svc.Spec.Ports { - switch p.Name { - case render.APIServerPortName: - Expect(p.Port).To(Equal(int32(443))) - Expect(p.TargetPort.IntVal).To(Equal(apiServerPort.ContainerPort)) - servicesFound++ - case render.QueryServerPortName: - Expect(p.Port).To(Equal(int32(8080))) - Expect(p.TargetPort.IntVal).To(Equal(queryServerPort.ContainerPort)) - servicesFound++ - case render.L7AdmissionControllerPortName: - Expect(p.Port).To(Equal(int32(6443))) - Expect(p.TargetPort.IntVal).To(Equal(l7AdmCtrlPort.ContainerPort)) - servicesFound++ - } - } - Expect(servicesFound).To(Equal(3)) - }) - - It("should override a ControlPlaneNodeSelector when specified", func() { - cfg.Installation.ControlPlaneNodeSelector = map[string]string{"nodeName": "control01"} - - cfg.APIServer.APIServerDeployment = &operatorv1.APIServerDeployment{ - Spec: &operatorv1.APIServerDeploymentSpec{ - Template: &operatorv1.APIServerDeploymentPodTemplateSpec{ - Spec: &operatorv1.APIServerDeploymentPodSpec{ - NodeSelector: map[string]string{ - "custom-node-selector": "value", - }, - }, - }, - }, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - // nodeSelectors are merged - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(2)) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) - Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("custom-node-selector", "value")) - }) - - It("should override ControlPlaneTolerations when specified", func() { - cfg.Installation.ControlPlaneTolerations = rmeta.TolerateControlPlane - - tol := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - Effect: corev1.TaintEffectNoExecute, - } + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/k8sapi" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/podaffinity" + rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/test" - cfg.APIServer.APIServerDeployment = &operatorv1.APIServerDeployment{ - Spec: &operatorv1.APIServerDeploymentSpec{ - Template: &operatorv1.APIServerDeploymentPodTemplateSpec{ - Spec: &operatorv1.APIServerDeploymentPodSpec{ - Tolerations: []corev1.Toleration{tol}, - }, - }, - }, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(ok).To(BeTrue()) - Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) - Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(tol)) - }) + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + apiregv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) - It("should disable ValidatingAdmissionPolicy on older k8s versions", func() { - cfg.KubernetesVersion = &common.VersionInfo{ - Major: 1, - Minor: 28, - } - component, err := render.APIServer(cfg) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() - d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(d.Spec.Template.Spec.Containers[0].Args).To(ConsistOf([]string{ - "--secure-port=5443", - "--tls-private-key-file=/calico-apiserver-certs/tls.key", - "--tls-cert-file=/calico-apiserver-certs/tls.crt", - "--audit-policy-file=/etc/tigera/audit/policy.conf", - "--audit-log-path=/var/log/calico/audit/tsee-audit.log", - "--enable-validating-admission-policy=false", - })) - }) - }) -}) +// apiServerObjects renders the base (OSS) API server component. The enterprise +// modifier and the Calico-variant cleanup are exercised in pkg/enterprise/apiserver; +// these tests cover the OSS render path, which the base render handles on its own +// (including the deletes it queues for itself). +func apiServerObjects(c render.Component) ([]client.Object, []client.Object) { + return c.Objects() +} func verifyAPIService(service *apiregv1.APIService, enterprise bool, clusterDomain string) { Expect(service.Name).To(Equal("v3.projectcalico.org")) @@ -1610,452 +74,6 @@ func verifyAPIService(service *apiregv1.APIService, enterprise bool, clusterDoma test.VerifyCertSANs(ca, expectedDNSNames...) } -func validateTunnelSecret(voltronSecret *corev1.Secret) { - var newCert *x509.Certificate - - cert := voltronSecret.Data[corev1.TLSCertKey] - key := voltronSecret.Data[corev1.TLSPrivateKeyKey] - _, err := tls.X509KeyPair(cert, key) - Expect(err).ShouldNot(HaveOccurred()) - - roots := x509.NewCertPool() - ok := roots.AppendCertsFromPEM([]byte(cert)) - Expect(ok).To(BeTrue()) - - block, _ := pem.Decode([]byte(cert)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(block).To(Not(BeNil())) - - newCert, err = x509.ParseCertificate(block.Bytes) - Expect(err).ShouldNot(HaveOccurred()) - - opts := x509.VerifyOptions{ - DNSName: "voltron", - Roots: roots, - } - - _, err = newCert.Verify(opts) - Expect(err).ShouldNot(HaveOccurred()) - - opts = x509.VerifyOptions{ - DNSName: "voltron", - Roots: x509.NewCertPool(), - CurrentTime: time.Now().Add(crypto.DefaultCACertificateLifetimeDuration + 24*time.Hour), - } - _, err = newCert.Verify(opts) - Expect(err).Should(HaveOccurred()) -} - -var ( - uiUserPolicyRules = []rbacv1.PolicyRule{ - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - "", - }, - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "namespaces", - "globalnetworksets", - "networksets", - "managedclusters", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "policyrecommendationscopes", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"get", "list", "watch"}, - ResourceNames: []string{"cluster-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"user-settings"}, - }, - { - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", - }, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"gatewayapis"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "globalwafpolicies", - "globalwafplugins", - "globalwafvalidationpolicies", - "wafpolicies", - "wafplugins", - "wafvalidationpolicies", - }, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list"}, - }, - } - networkAdminPolicyRules = []rbacv1.PolicyRule{ - { - APIGroups: []string{ - "projectcalico.org", - "networking.k8s.io", - "extensions", - }, - Resources: []string{ - "tiers", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedkubernetesnetworkpolicies", - "globalnetworksets", - "networksets", - "managedclusters", - "packetcaptures", - "policyrecommendationscopes", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{ - "policy.networking.k8s.io", - }, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"packetcaptures/files"}, - Verbs: []string{"get", "delete"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"watch", "list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{"coreruleset-default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "https:calico-api:8080", "calico-node-prometheus:9090", - }, - Verbs: []string{"get", "create"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports"}, - Verbs: []string{"*"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreports/status"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "alertexceptions", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "globalthreatfeeds", - "globalthreatfeeds/status", - "securityeventwebhooks", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get", "patch", "update"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", - }, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "compliances", "intrusiondetections"}, - Verbs: []string{"get", "update", "patch", "create", "delete"}, - }, - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"gatewayapis"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "globalwafpolicies", - "globalwafplugins", - "globalwafvalidationpolicies", - "wafpolicies", - "wafplugins", - "wafvalidationpolicies", - }, - Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "watch", "patch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "patch", "create", "delete"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - ResourceNames: []string{"webhooks-secret"}, - Verbs: []string{"patch"}, - }, - // Write access to the switch, ungated so it can be used to turn the feature on. - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{rbacmanagement.ConfigMapName}, - Verbs: []string{"get", "list", "watch", "update", "patch", "delete"}, - }, - } - - // rbacManagementNetworkAdminRules are the extra tigera-network-admin rules added - // while the feature is enabled. - rbacManagementNetworkAdminRules = []rbacv1.PolicyRule{ - { - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "roles"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterrolebindings", "rolebindings"}, - Verbs: []string{"get", "list", "watch", "create", "update", "delete"}, - }, - } -) - var _ = Describe("API server rendering tests (Calico)", func() { var instance *operatorv1.InstallationSpec var apiserver *operatorv1.APIServerSpec @@ -2119,7 +137,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) rtest.ExpectResources(resources, expectedResources) rtest.ExpectResourceInList(deleteResources, "allow-apiserver", "calico-system", "networking.k8s.io", "v1", "NetworkPolicy") @@ -2215,7 +233,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) // Should not include deployment, service, SA, or PDB. Expect(rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment")).To(BeNil()) @@ -2273,7 +291,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, deleteResources := component.Objects() + resources, deleteResources := apiServerObjects(component) // Should render the correct resources. By("Checking each expected resource is actually rendered") @@ -2310,7 +328,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(1)) Expect(d.Spec.Template.Spec.NodeSelector).To(HaveKeyWithValue("nodeName", "control01")) @@ -2326,7 +344,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { cfg.Installation.ControlPlaneTolerations = []corev1.Toleration{tol} component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElements(append(rmeta.TolerateControlPlane, tol))) }) @@ -2340,7 +358,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2353,7 +371,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { cfg.ForceHostNetwork = true component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Strategy.Type).To(Equal(appsv1.RecreateDeploymentStrategyType)) }) @@ -2366,7 +384,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2383,7 +401,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploymentResource := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment") Expect(deploymentResource).ToNot(BeNil()) @@ -2398,7 +416,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2411,7 +429,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2425,7 +443,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - _, _ = component.Objects() + _, _ = apiServerObjects(component) }) It("should render host networked with TKG provider", func() { @@ -2436,7 +454,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) deploy, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2542,13 +560,10 @@ var _ = Describe("API server rendering tests (Calico)", func() { kp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) cfg.TLSKeyPair = kp - qskp, err := certificateManager.GetOrCreateKeyPair(cli, render.CalicoAPIServerTLSSecretName, common.OperatorNamespace(), dnsNames) - Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - cfg.QueryServerTLSKeyPairCertificateManagementOnly = qskp component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d, ok := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(ok).To(BeTrue()) @@ -2586,7 +601,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { Expect(d.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort).To(Equal(apiServerPort.ContainerPort)) Expect(d.Spec.Template.Spec.Containers[0].Args[0]).To(ContainSubstring(fmt.Sprintf("--secure-port=%d", apiServerPort.ContainerPort))) - Expect(d.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + Expect(d.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(d.Spec.Template.Spec.InitContainers[0].Name).To(Equal("calico-apiserver-certs-key-cert-provisioner")) Expect(d.Spec.Template.Spec.InitContainers[0].Resources).To(Equal(rr2)) @@ -2624,7 +639,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) // nodeSelectors are merged Expect(d.Spec.Template.Spec.NodeSelector).To(HaveLen(2)) @@ -2654,7 +669,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).To(BeNil(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d.Spec.Template.Spec.Tolerations).To(HaveLen(1)) Expect(d.Spec.Template.Spec.Tolerations).To(ConsistOf(tol)) @@ -2666,7 +681,7 @@ var _ = Describe("API server rendering tests (Calico)", func() { component, err := render.APIServer(cfg) Expect(err).NotTo(HaveOccurred(), "Expected APIServer to create successfully %s", err) Expect(component.ResolveImages(nil)).NotTo(HaveOccurred()) - resources, _ := component.Objects() + resources, _ := apiServerObjects(component) d := rtest.GetResource(resources, "calico-apiserver", "calico-system", "apps", "v1", "Deployment").(*appsv1.Deployment) Expect(d).NotTo(BeNil()) Expect(d.Spec.Template.Spec.Tolerations).To(ContainElement(corev1.Toleration{ @@ -2677,106 +692,4 @@ var _ = Describe("API server rendering tests (Calico)", func() { })) }) }) - - Context("multi-tenant", func() { - BeforeEach(func() { - cfg.MultiTenant = true - cfg.ManagementCluster = &operatorv1.ManagementCluster{Spec: operatorv1.ManagementClusterSpec{Address: "example.com:1234"}} - cfg.Installation = &operatorv1.InstallationSpec{ - ControlPlaneReplicas: ptr.To(int32(2)), - Registry: "testregistry.com/", - Variant: operatorv1.CalicoEnterprise, - } - }) - - It("should not install tigera-network-admin and tigera-ui-user", func() { - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - // Expect no UISettings / UISettingsGroups to be installed. - resources, _ := component.Objects() - obj := rtest.GetResource(resources, "tigera-network-admin", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") - Expect(obj).To(BeNil()) - obj = rtest.GetResource(resources, "tigera-ui-user", "", "rbac.authorization.k8s.io", "v1", "ClusterRole") - Expect(obj).To(BeNil()) - }) - - It("should create a cluster role that get managed clusters", func() { - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - resources, _ := component.Objects() - managedClusterAccessRole := rtest.GetResource(resources, - render.MultiTenantManagedClustersAccessClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) - expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{"get"}, - }, - } - Expect(managedClusterAccessRole.Rules).To(ContainElements(expectedManagedClusterAccessRules)) - }) - - It("should create a cluster role for watching managed clusters", func() { - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - resources, _ := component.Objects() - managedClusterAccessRole := rtest.GetResource(resources, - render.ManagedClustersWatchClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) - expectedManagedClusterAccessRules := []rbacv1.PolicyRule{ - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"managedclusters"}, - Verbs: []string{"get", "list", "watch"}, - }, - } - Expect(managedClusterAccessRole.Rules).To(ContainElements(expectedManagedClusterAccessRules)) - }) - - It("should bind the calico-apiserver ClusterRole only to the calico-system service account", func() { - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - resources, _ := component.Objects() - crb := rtest.GetResource(resources, - render.APIServerName, "", rbacv1.GroupName, "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Name).To(Equal(render.APIServerName)) - // The aggregated API server always runs in calico-system, even in multi-tenant mode - its - // full-privilege ClusterRole must not be bound to tenant service accounts. - Expect(crb.Subjects).To(ConsistOf(rbacv1.Subject{ - Kind: "ServiceAccount", - Name: render.APIServerServiceAccountName, - Namespace: render.APIServerNamespace, - })) - }) - - It("should grant each tenant's calico-apiserver service account least-privilege Linseed access", func() { - cfg.BindingNamespaces = []string{"tenant-a", "tenant-b"} - component, err := render.APIServer(cfg) - Expect(err).NotTo(HaveOccurred()) - - resources, _ := component.Objects() - - // A dedicated, Linseed-only ClusterRole. - role := rtest.GetResource(resources, - render.APIServerLinseedAccessClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(role.Rules).To(ConsistOf(rbacv1.PolicyRule{ - APIGroups: []string{"linseed.tigera.io"}, - Resources: []string{"policyactivity"}, - Verbs: []string{"get"}, - })) - - // A single ClusterRoleBinding with one calico-apiserver ServiceAccount subject per tenant namespace. - // Linseed authorizes with a cluster-scoped SubjectAccessReview, so this must be a ClusterRoleBinding. - crb := rtest.GetResource(resources, - render.APIServerLinseedAccessClusterRoleName, "", rbacv1.GroupName, "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(crb.RoleRef.Name).To(Equal(render.APIServerLinseedAccessClusterRoleName)) - Expect(crb.Subjects).To(ConsistOf( - rbacv1.Subject{Kind: "ServiceAccount", Name: render.APIServerServiceAccountName, Namespace: "tenant-a"}, - rbacv1.Subject{Kind: "ServiceAccount", Name: render.APIServerServiceAccountName, Namespace: "tenant-b"}, - )) - }) - }) }) diff --git a/pkg/render/component.go b/pkg/render/component.go index eea911fbe3..3fe9c48ef9 100644 --- a/pkg/render/component.go +++ b/pkg/render/component.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2024 Tigera, Inc. All rights reserved. +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,9 +15,10 @@ package render import ( + "sigs.k8s.io/controller-runtime/pkg/client" + operatorv1 "github.com/tigera/operator/api/v1" rmeta "github.com/tigera/operator/pkg/render/common/meta" - "sigs.k8s.io/controller-runtime/pkg/client" ) type Component interface { @@ -39,3 +40,58 @@ type Component interface { // that create pods. Return OSTypeAny means that no node selector should be set for the "kubernetes.io/os" label. SupportedOSType() rmeta.OSType } + +// A component that exposes an extension point hands a variant the config it rendered +// from. The accessor names differ per component, so an unrelated one can't match. +type ( + NodeComponent interface { + Component + NodeConfig() *NodeConfiguration + } + + TyphaComponent interface { + Component + TyphaConfig() *TyphaConfiguration + } + + WindowsComponent interface { + Component + WindowsConfig() *WindowsConfiguration + } + + GuardianComponent interface { + Component + GuardianConfig() *GuardianConfiguration + } + + GuardianPolicyComponent interface { + Component + GuardianPolicyConfig() *GuardianConfiguration + } + + APIServerComponent interface { + Component + APIServerConfig() *APIServerConfiguration + } + + APIServerPolicyComponent interface { + Component + APIServerPolicyConfig() *APIServerConfiguration + } +) + +// Component names, which key the image overrides a variant resolves through. +const ( + ComponentNameNode = "node" + + // ComponentNameCNIPlugins keys the upstream CNI plugins image. The node + // component renders the cni-plugins init container, so the image resolves + // through its own override key. + ComponentNameCNIPlugins = "cni-plugins" + + // The two windows images get their own keys, since one component renders both. + ComponentNameWindowsNodeImg = "windows-node-image" + ComponentNameWindowsCNIImg = "windows-cni-image" + + ComponentNameKubeControllers = "kube-controllers" +) diff --git a/pkg/render/containers.go b/pkg/render/containers.go new file mode 100644 index 0000000000..db82d004fe --- /dev/null +++ b/pkg/render/containers.go @@ -0,0 +1,58 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" +) + +// Container returns the named container in spec, init containers included. The +// returned pointer aliases spec. Use it for containers a component only renders +// under some configurations; for the rest, use MustContainer. +func Container(spec *corev1.PodSpec, name string) (*corev1.Container, bool) { + for i := range spec.Containers { + if spec.Containers[i].Name == name { + return &spec.Containers[i], true + } + } + for i := range spec.InitContainers { + if spec.InitContainers[i].Name == name { + return &spec.InitContainers[i], true + } + } + return nil, false +} + +// MustContainer returns the named container, panicking if it is absent. A modifier +// asking for a container that is always rendered and not finding one means render +// and the modifier have drifted apart, which no caller can recover from. +func MustContainer(spec *corev1.PodSpec, name string) *corev1.Container { + c, ok := Container(spec, name) + if !ok { + panic(fmt.Sprintf("BUG: no container named %q to modify", name)) + } + return c +} + +// MustContainers returns the named containers, panicking if any is absent. +func MustContainers(spec *corev1.PodSpec, names ...string) []*corev1.Container { + found := make([]*corev1.Container, 0, len(names)) + for _, name := range names { + found = append(found, MustContainer(spec, name)) + } + return found +} diff --git a/pkg/render/guardian.go b/pkg/render/guardian.go index 68678d6ea7..9679ba08d1 100644 --- a/pkg/render/guardian.go +++ b/pkg/render/guardian.go @@ -18,13 +18,9 @@ package render import ( "fmt" - "net" - "net/url" "golang.org/x/net/http/httpproxy" - operatorurl "github.com/tigera/operator/pkg/url" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" @@ -35,8 +31,6 @@ import ( v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" - operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" @@ -45,7 +39,6 @@ import ( "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -80,32 +73,34 @@ var ( ) func Guardian(cfg *GuardianConfiguration) Component { - return &GuardianComponent{ + return &guardianComponent{ cfg: cfg, } } +// GuardianPolicy renders the OSS guardian network policy. A variant may replace it +// with its own. The error return is always nil, and is kept for callers. func GuardianPolicy(cfg *GuardianConfiguration) (Component, error) { - var policies []client.Object + return &guardianPolicyComponent{cfg: cfg}, nil +} - guardianAccessPolicy, err := guardianCalicoSystemPolicy(cfg) - if err != nil { - return nil, err - } - if guardianAccessPolicy != nil { - policies = []client.Object{ - guardianAccessPolicy, - } - } +type guardianPolicyComponent struct { + cfg *GuardianConfiguration +} - return NewPassthrough( - policies, - []client.Object{ - // allow-tigera Tier was renamed to calico-system - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("guardian-access", GuardianNamespace), - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", GuardianNamespace), - }, - ), nil +func (c *guardianPolicyComponent) ResolveImages(*operatorv1.ImageSet) error { return nil } +func (c *guardianPolicyComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeAny } +func (c *guardianPolicyComponent) Ready() bool { return true } +func (c *guardianPolicyComponent) GuardianPolicyConfig() *GuardianConfiguration { + return c.cfg +} + +func (c *guardianPolicyComponent) Objects() ([]client.Object, []client.Object) { + return []client.Object{ossNetworkPolicy(c.cfg)}, []client.Object{ + // allow-tigera Tier was renamed to calico-system + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("guardian-access", GuardianNamespace), + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", GuardianNamespace), + } } // GuardianConfiguration contains all the config information needed to render the component. @@ -132,12 +127,37 @@ type GuardianConfiguration struct { Version string } -type GuardianComponent struct { +// GuardianRenderData is the variant-specific Guardian input a controller extension +// computes during reconcile and stashes in Inputs.Extension. The +// clusterconnection controller reads it back to fill GuardianConfiguration without +// depending on the extension: when present it carries the enterprise values +// (the management-cluster version and the license-gated egress policy flag) and +// signals that the controller should not create the OSS Guardian client keypair. +// It lives in render so the controller can read it generically. +type GuardianRenderData struct { + // Version is the managed cluster version reported by ClusterInformation + // (CNXVersion for Enterprise, CalicoVersion for the OSS default). + Version string + + // IncludeEgressNetworkPolicy enables the domain-based egress rules in the + // Guardian policy, gated on an Enterprise license feature. + IncludeEgressNetworkPolicy bool +} + +// GuardianRenderDataFromInputs returns the GuardianRenderData a controller +// extension stashed in the render inputs, and whether it was present. Absent +// means the OSS path: the controller applies its own defaults. +func GuardianRenderDataFromInputs(ri Inputs) (GuardianRenderData, bool) { + data, ok := ri.Extension.(GuardianRenderData) + return data, ok +} + +type guardianComponent struct { cfg *GuardianConfiguration calicoImage string } -func (c *GuardianComponent) ResolveImages(is *operatorv1.ImageSet) error { +func (c *guardianComponent) ResolveImages(is *operatorv1.ImageSet) error { reg := c.cfg.Installation.Registry path := c.cfg.Installation.ImagePath prefix := c.cfg.Installation.ImagePrefix @@ -146,45 +166,32 @@ func (c *GuardianComponent) ResolveImages(is *operatorv1.ImageSet) error { return err } -func (c *GuardianComponent) SupportedOSType() rmeta.OSType { +func (c *guardianComponent) GuardianConfig() *GuardianConfiguration { + return c.cfg +} + +func (c *guardianComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } -func (c *GuardianComponent) Objects() ([]client.Object, []client.Object) { +func (c *guardianComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{ - // common RBAC for EE and OSS c.serviceAccount(), c.clusterRole(), c.clusterRoleBinding(), - } - - if c.cfg.Installation.Variant.IsEnterprise() { - // Enterprise-specific RBAC and settings - objs = append(objs, - c.secretsRole(), - c.secretRoleBinding(), - // Install default UI settings for this managed cluster. - managerClusterWideSettingsGroup(), - managerUserSpecificSettingsGroup(), - managerClusterWideTigeraLayer(), - managerClusterWideDefaultView(), - ) - } - - objs = append(objs, c.deployment(), c.service(), secret.CopyToNamespace(GuardianNamespace, c.cfg.TunnelSecret)[0], - ) + } return objs, deprecatedObjects() } -func (c *GuardianComponent) Ready() bool { +func (c *guardianComponent) Ready() bool { return true } -func (c *GuardianComponent) service() *corev1.Service { +func (c *guardianComponent) service() *corev1.Service { ports := []corev1.ServicePort{ { Name: "https", @@ -197,28 +204,6 @@ func (c *GuardianComponent) service() *corev1.Service { }, } - if c.cfg.Installation.Variant.IsEnterprise() { - ports = append(ports, - corev1.ServicePort{ - Name: "elasticsearch", - Port: 9200, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - Protocol: corev1.ProtocolTCP, - }, - corev1.ServicePort{ - Name: "kibana", - Port: 5601, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: 8080, - }, - Protocol: corev1.ProtocolTCP, - }, - ) - } return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: GuardianServiceName, @@ -233,95 +218,50 @@ func (c *GuardianComponent) service() *corev1.Service { } } -func (c *GuardianComponent) serviceAccount() *corev1.ServiceAccount { +func (c *guardianComponent) serviceAccount() *corev1.ServiceAccount { return &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: GuardianServiceAccountName, Namespace: GuardianNamespace}, } } -func (c *GuardianComponent) clusterRole() *rbacv1.ClusterRole { - var policyRules []rbacv1.PolicyRule - if c.cfg.Installation.Variant.IsEnterprise() { - impersonation := c.cfg.ManagementClusterConnection.Spec.Impersonation - if impersonation != nil { - if impersonation.Users != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"users"}, - ResourceNames: impersonation.Users, - Verbs: []string{"impersonate"}, - }) - } - if impersonation.Groups != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"groups"}, - ResourceNames: impersonation.Groups, - Verbs: []string{"impersonate"}, - }) - } - if impersonation.ServiceAccounts != nil { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - ResourceNames: impersonation.ServiceAccounts, - Verbs: []string{"impersonate"}, - }) - } - } - - policyRules = append(policyRules, rulesForManagementClusterRequests(c.cfg.OpenShift)...) - - if c.cfg.OpenShift { - policyRules = append(policyRules, rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.NonRootV2}, - }) - } - } else { - policyRules = append(policyRules, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces", "services", "pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"apps"}, - Resources: []string{"deployments", "replicasets", "statefulsets", "daemonsets"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list", "watch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "clusterinformations", - "tiers", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - "stagedglobalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "tier.stagedkubernetesnetworkpolicies", - "networkpolicies", - "tier.networkpolicies", - "globalnetworkpolicies", - "tier.globalnetworkpolicies", - "globalnetworksets", - "networksets", - }, - Verbs: []string{"get", "list", "watch"}, +func (c *guardianComponent) clusterRole() *rbacv1.ClusterRole { + policyRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"namespaces", "services", "pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{"deployments", "replicasets", "statefulsets", "daemonsets"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"projectcalico.org"}, + Resources: []string{ + "clusterinformations", + "tiers", + "stagednetworkpolicies", + "tier.stagednetworkpolicies", + "stagedglobalnetworkpolicies", + "tier.stagedglobalnetworkpolicies", + "stagedkubernetesnetworkpolicies", + "tier.stagedkubernetesnetworkpolicies", + "networkpolicies", + "tier.networkpolicies", + "globalnetworkpolicies", + "tier.globalnetworkpolicies", + "globalnetworksets", + "networksets", }, - ) + Verbs: []string{"get", "list", "watch"}, + }, } return &rbacv1.ClusterRole{ @@ -333,7 +273,7 @@ func (c *GuardianComponent) clusterRole() *rbacv1.ClusterRole { } } -func (c *GuardianComponent) clusterRoleBinding() *rbacv1.ClusterRoleBinding { +func (c *guardianComponent) clusterRoleBinding() *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ @@ -354,48 +294,7 @@ func (c *GuardianComponent) clusterRoleBinding() *rbacv1.ClusterRoleBinding { } } -// secretRole creates a Role that allows the management cluster to provision secrets to the tigera-operator Namespace. -// This is used to push secrets used by the managed cluster to access / authenticate with the management cluster. -func (c *GuardianComponent) secretsRole() *rbacv1.Role { - return &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianSecretsRole, - Namespace: common.OperatorNamespace(), - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"create", "delete", "deletecollection", "update"}, - }, - }, - } -} - -func (c *GuardianComponent) secretRoleBinding() *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianSecretsRoleBindingName, - Namespace: common.OperatorNamespace(), - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: GuardianSecretsRole, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: GuardianServiceAccountName, - Namespace: GuardianNamespace, - }, - }, - } -} - -func (c *GuardianComponent) deployment() *appsv1.Deployment { +func (c *guardianComponent) deployment() *appsv1.Deployment { var replicas int32 = 1 tolerations := append(c.cfg.Installation.ControlPlaneTolerations, rmeta.TolerateCriticalAddonsAndControlPlane...) @@ -440,7 +339,7 @@ func (c *GuardianComponent) deployment() *appsv1.Deployment { return d } -func (c *GuardianComponent) volumes() []corev1.Volume { +func (c *guardianComponent) volumes() []corev1.Volume { volumes := []corev1.Volume{ c.cfg.TrustedCertBundle.Volume(), { @@ -458,7 +357,7 @@ func (c *GuardianComponent) volumes() []corev1.Volume { return volumes } -func (c *GuardianComponent) container() []corev1.Container { +func (c *guardianComponent) container() []corev1.Container { envVars := []corev1.EnvVar{ {Name: "GUARDIAN_PORT", Value: "9443"}, {Name: "GUARDIAN_LOGLEVEL", Value: "INFO"}, @@ -468,14 +367,6 @@ func (c *GuardianComponent) container() []corev1.Container { } envVars = append(envVars, c.cfg.Installation.Proxy.EnvVars()...) - if c.cfg.Installation.Variant.IsEnterprise() { - envVars = append(envVars, - corev1.EnvVar{Name: "GUARDIAN_PACKET_CAPTURE_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - corev1.EnvVar{Name: "GUARDIAN_PROMETHEUS_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - corev1.EnvVar{Name: "GUARDIAN_QUERYSERVER_CA_BUNDLE_PATH", Value: c.cfg.TrustedCertBundle.MountPath()}, - ) - } - if c.cfg.GuardianClientKeyPair != nil { envVars = append(envVars, corev1.EnvVar{ @@ -523,7 +414,7 @@ func (c *GuardianComponent) container() []corev1.Container { } } -func (c *GuardianComponent) volumeMounts() []corev1.VolumeMount { +func (c *guardianComponent) volumeMounts() []corev1.VolumeMount { volumeMounts := append( c.cfg.TrustedCertBundle.VolumeMounts(c.SupportedOSType()), corev1.VolumeMount{Name: GuardianVolumeName, MountPath: "/certs/", ReadOnly: true}, @@ -534,7 +425,7 @@ func (c *GuardianComponent) volumeMounts() []corev1.VolumeMount { return volumeMounts } -func (c *GuardianComponent) annotations() map[string]string { +func (c *guardianComponent) annotations() map[string]string { annotations := c.cfg.TrustedCertBundle.HashAnnotations() annotations["hash.operator.tigera.io/tigera-managed-cluster-connection"] = rmeta.AnnotationHash(c.cfg.TunnelSecret.Data) @@ -585,173 +476,6 @@ func ossNetworkPolicy(cfg *GuardianConfiguration) *v3.NetworkPolicy { } } -func guardianCalicoSystemPolicy(cfg *GuardianConfiguration) (*v3.NetworkPolicy, error) { - if !cfg.Installation.Variant.IsEnterprise() { - return ossNetworkPolicy(cfg), nil - } - - egressRules := []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: PacketCaptureEntityRule, - }, - } - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.OpenShift) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.KubeAPIServerEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.PrometheusEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: TigeraAPIServerEntityRule, - }, - }...) - - // The loop below creates an egress rule for each unique destination that the Guardian pods connect to. If there are - // multiple guardian pods and their proxy settings differ, then there are multiple destinations that must have egress allowed. - allowedDestinations := map[string]bool{} - processedPodProxies := ProcessPodProxies(cfg.PodProxies) - for _, podProxyConfig := range processedPodProxies { - var proxyURL *url.URL - var err error - if podProxyConfig != nil && podProxyConfig.HTTPSProxy != "" { - targetURL := &url.URL{ - // The scheme should be HTTPS, as we are establishing an mTLS session with the target. - Scheme: "https", - - // We expect `target` to be of the form host:port. - Host: cfg.URL, - } - - proxyURL, err = podProxyConfig.ProxyFunc()(targetURL) - if err != nil { - return nil, err - } - } - - var tunnelDestinationHostPort string - if proxyURL != nil { - proxyHostPort, err := operatorurl.ParseHostPortFromHTTPProxyURL(proxyURL) - if err != nil { - return nil, err - } - - tunnelDestinationHostPort = proxyHostPort - } else { - // cfg.URL has host:port form - tunnelDestinationHostPort = cfg.URL - } - - // Check if we've already created an egress rule for this destination. - if allowedDestinations[tunnelDestinationHostPort] { - continue - } - - host, port, err := net.SplitHostPort(tunnelDestinationHostPort) - if err != nil { - return nil, err - } - parsedPort, err := numorstring.PortFromString(port) - if err != nil { - return nil, err - } - parsedIp := net.ParseIP(host) - if parsedIp == nil { - // Domain-based egress rules require the EgressAccessControl license feature. - if !cfg.IncludeEgressNetworkPolicy { - continue - } - // Assume host is a valid hostname. - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Domains: []string{host}, - Ports: []numorstring.Port{parsedPort}, - }, - }) - allowedDestinations[tunnelDestinationHostPort] = true - - } else { - var netSuffix string - if parsedIp.To4() != nil { - netSuffix = "/32" - } else { - netSuffix = "/128" - } - - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Nets: []string{parsedIp.String() + netSuffix}, - Ports: []numorstring.Port{parsedPort}, - }, - }) - allowedDestinations[tunnelDestinationHostPort] = true - } - } - - egressRules = append(egressRules, v3.Rule{Action: v3.Pass}) - - guardianIngressDestinationEntityRule := v3.EntityRule{Ports: networkpolicy.Ports(GuardianTargetPort)} - var ingressRules []v3.Rule - if cfg.Installation.Variant.IsEnterprise() { - ingressRules = append(ingressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: FluentBitSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: IntrusionDetectionSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: IntrusionDetectionInstallerSourceEntityRule, - Destination: guardianIngressDestinationEntityRule, - }, - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: guardianIngressDestinationEntityRule, - }, - }...) - } - - policy := &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: GuardianPolicyName, - Namespace: GuardianNamespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(GuardianName), - Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - Ingress: ingressRules, - Egress: egressRules, - }, - } - - return policy, nil -} - func ProcessPodProxies(podProxies []*httpproxy.Config) []*httpproxy.Config { // If pod proxies are empty, then pod proxy resolution has not yet occurred. // Assume that a single Guardian pod is running without a proxy. @@ -766,304 +490,6 @@ func GuardianService(clusterDomain string) string { return fmt.Sprintf("https://%s.%s.svc.%s:%d", GuardianServiceName, GuardianNamespace, clusterDomain, 443) } -// rulesForManagementClusterRequests returns the set of RBAC rules needed by Guardian in order to -// satisfy requests from the management cluster over the tunnel. -func rulesForManagementClusterRequests(isOpenShift bool) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - // Common rules required to handle requests from multiple components in the management cluster. - { - // ID uses read-only permissions and kube-controllers uses both read and write verbs. - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - // Allows Linseed to watch namespaces before copying its token. - // Also enables PolicyRecommendation to watch namespaces, - // and Manager/kube-controllers to list them. - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // kube-controllers watches Nodes to monitor for deletions. - // Manager performs a list operation on Nodes. - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // kube-controllers watches Pods to verify existence for IPAM garbage collection. - // Manager performs get operations on Pods. - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Manager uses list; kube-controllers uses 'get', 'list', 'watch', 'update'. - APIGroups: []string{""}, - Resources: []string{"services"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - // Needed by kube-controllers to validate licenses; also used by ID. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch"}, - }, - { - // Manager uses list; PolicyRecommendation & ID uses all verbs. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalnetworksets", - "networkpolicies", - "tier.networkpolicies", - "stagednetworkpolicies", - "tier.stagednetworkpolicies", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - { - // Manager uses list; PolicyRecommendation uses all verbs. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - // Rules needed by guardian to handle manager authorization reviews. - { - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings", "roles", "rolebindings"}, - Verbs: []string{"list", "get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettings", "uisettingsgroups"}, - Verbs: []string{"list", "get"}, - }, - - // Rules needed by guardian to handle other manager requests. - { - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"list"}, - }, - { - // Allow query server talk to Prometheus via the manager user. - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{ - "calico-node-prometheus:9090", - "https:calico-api:8080", - }, - Verbs: []string{"create", "get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"daemonsets", "replicasets", "statefulsets"}, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"authentication.k8s.io"}, - Resources: []string{"tokenreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"authorization.k8s.io"}, - Resources: []string{"subjectaccessreviews"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"policy.networking.k8s.io"}, - Resources: []string{ - "clusternetworkpolicies", - "adminnetworkpolicies", - "baselineadminnetworkpolicies", - }, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"alertexceptions"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - ResourceNames: []string{"default"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalnetworkpolicies", - "networksets", - "stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "tier.globalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - }, - Verbs: []string{"list"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"list"}, - }, - - // Rules needed by guardian to handle policy recommendation requests. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "policyrecommendationscopes", - "policyrecommendationscopes/status", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - - // Rules needed by guardian to handle calico-kube-controller requests. - { - // Nodes are watched to monitor for deletions. - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"services/status"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - // Needs to manage hostendpoints. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"hostendpoints"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - { - // Needs access to update clusterinformations. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - // Needs to manipulate kubecontrollersconfiguration, which contains its config. - // It creates a default if none exists, and updates status as well. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"kubecontrollersconfigurations"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"tiers"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{"crd.projectcalico.org", "projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"remoteclusterconfigurations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"create", "get", "list", "update", "watch"}, - }, - { - // Grant permissions to access ClusterInformation resources in managed clusters. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"clusterinformations"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"usage.tigera.io"}, - Resources: []string{"licenseusagereports"}, - Verbs: []string{"create", "delete", "get", "list", "update", "watch"}, - }, - - // Rules needed by guardian to handle Intrusion detection requests. - { - APIGroups: []string{""}, - Resources: []string{"podtemplates"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{"deployments"}, - Verbs: []string{"get"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"alertexceptions"}, - Verbs: []string{"get", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"securityeventwebhooks"}, - Verbs: []string{"get", "list", "update", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "globalalerts", - "globalalerts/status", - "globalthreatfeeds", - "globalthreatfeeds/status", - }, - Verbs: []string{"create", "delete", "get", "list", "patch", "update", "watch"}, - }, - // Rules needed to fetch the compliance reports - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"globalreporttypes", "globalreports"}, - Verbs: []string{"get", "list", "watch"}, - }, - } - - // Rules needed by policy recommendation in openshift. - if isOpenShift { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{securitycontextconstraints.HostNetworkV2}, - }, - ) - } - - return rules -} - func deprecatedObjects() []client.Object { return []client.Object{ // All the Guardian objects were moved to "calico-system" circa Calico v3.30, and so the legacy tigera-guardian diff --git a/pkg/render/guardian_test.go b/pkg/render/guardian_test.go index 76f7fa56ea..c99c1e485f 100644 --- a/pkg/render/guardian_test.go +++ b/pkg/render/guardian_test.go @@ -20,8 +20,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - netv1 "k8s.io/api/networking/v1" - rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -33,398 +31,77 @@ import ( operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" - "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/render" - rmeta "github.com/tigera/operator/pkg/render/common/meta" - "github.com/tigera/operator/pkg/render/common/networkpolicy" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/testutils" ) -var _ = Describe("Rendering tests", func() { - var cfg *render.GuardianConfiguration - var g render.Component - var resources []client.Object - var deleteResources []client.Object - - createGuardianConfig := func(i operatorv1.InstallationSpec, addr string, openshift bool) *render.GuardianConfiguration { - i.Variant = operatorv1.CalicoEnterprise - secret := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: render.GuardianSecretName, - Namespace: common.OperatorNamespace(), - }, - Data: map[string][]byte{ - "cert": []byte("foo"), - "key": []byte("bar"), - }, - } - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - - certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - bundle := certificateManager.CreateTrustedBundle() - - return &render.GuardianConfiguration{ - URL: addr, - PullSecrets: []*corev1.Secret{{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: "pull-secret", - Namespace: common.OperatorNamespace(), - }, - }}, - Installation: &i, - TunnelSecret: secret, - TrustedCertBundle: bundle, - OpenShift: openshift, - ManagementClusterConnection: &operatorv1.ManagementClusterConnection{}, - IncludeEgressNetworkPolicy: true, - } +// guardianObjects renders the base guardian component. The enterprise modifier is +// exercised in the pkg/enterprise/guardian tests; these tests cover the OSS render +// path, which never runs the modifier. +func guardianObjects(cfg *render.GuardianConfiguration) []client.Object { + g := render.Guardian(cfg) + ExpectWithOffset(1, g.ResolveImages(nil)).To(BeNil()) + objs, _ := g.Objects() + return objs +} + +func newGuardianConfig(addr string) *render.GuardianConfiguration { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: render.GuardianSecretName, + Namespace: common.OperatorNamespace(), + }, + Data: map[string][]byte{ + "cert": []byte("foo"), + "key": []byte("bar"), + }, } + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + return &render.GuardianConfiguration{ + URL: addr, + Installation: &operatorv1.InstallationSpec{Registry: "my-reg/"}, + TunnelSecret: secret, + TrustedCertBundle: certificateManager.CreateTrustedBundle(), + ManagementClusterConnection: &operatorv1.ManagementClusterConnection{}, + IncludeEgressNetworkPolicy: true, + } +} - Context("Guardian component", func() { - renderGuardian := func(i operatorv1.InstallationSpec) { - cfg = createGuardianConfig(i, "127.0.0.1:1234", false) - g = render.Guardian(cfg) - Expect(g.ResolveImages(nil)).To(BeNil()) - resources, deleteResources = g.Objects() - } - - BeforeEach(func() { - renderGuardian(operatorv1.InstallationSpec{Registry: "my-reg/"}) - }) - - It("should render all resources for a managed cluster", func() { - expectedResources := []client.Object{ - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceAccountName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianClusterRoleBindingName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRole, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretsRoleBindingName, Namespace: "tigera-operator"}, TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianDeploymentName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianServiceName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: ""}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: render.GuardianSecretName, Namespace: render.GuardianNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}}, - &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, - &v3.UISettingsGroup{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerUserSettings}, TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}}, - &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsLayerTigera}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, - &v3.UISettings{ObjectMeta: metav1.ObjectMeta{Name: render.ManagerClusterSettingsViewDefault}, TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}}, - } - - expectedDeleteResources := []client.Object{ - &corev1.Namespace{TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, - &rbacv1.ClusterRole{TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, - &rbacv1.ClusterRoleBinding{TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{Name: "tigera-guardian"}}, - &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"}}, - &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager", Namespace: "tigera-manager"}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, - &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-role"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "tigera-manager-binding"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &netv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "guardian", Namespace: "calico-system"}, TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "networking.k8s.io/v1"}}, - } - - rtest.ExpectResources(resources, expectedResources) - rtest.ExpectResources(deleteResources, expectedDeleteResources) - - deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal("my-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeFalse()) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsGroup).To(BeEquivalentTo(10001)) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsNonRoot).To(BeTrue()) - Expect(*deployment.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser).To(BeEquivalentTo(10001)) - Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.SeccompProfile).To(Equal( - &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - })) - Expect(deployment.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities).To(Equal( - &corev1.Capabilities{ - Drop: []corev1.Capability{"ALL"}, - }, - )) - }) - - It("should render controlPlaneTolerations", func() { - t := corev1.Toleration{ - Key: "foo", - Operator: corev1.TolerationOpEqual, - Value: "bar", - } - renderGuardian(operatorv1.InstallationSpec{ - ControlPlaneTolerations: []corev1.Toleration{t}, - }) - deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(deployment.Spec.Template.Spec.Tolerations).Should(ContainElements(append(rmeta.TolerateCriticalAddonsAndControlPlane, t))) - }) - - It("should render toleration on GKE", func() { - renderGuardian(operatorv1.InstallationSpec{ - KubernetesProvider: operatorv1.ProviderGKE, - }) - deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(deployment).NotTo(BeNil()) - Expect(deployment.Spec.Template.Spec.Tolerations).To(ContainElements(corev1.Toleration{ - Key: "kubernetes.io/arch", - Operator: corev1.TolerationOpEqual, - Value: "arm64", - Effect: corev1.TaintEffectNoSchedule, - })) - }) - - It("should render guardian with unlimited impersonation", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ - Spec: operatorv1.ManagementClusterConnectionSpec{ - Impersonation: &operatorv1.Impersonation{ - Users: []string{}, - Groups: []string{}, - ServiceAccounts: []string{}, - }, - }, - } - - g := render.Guardian(cfg) - resources, _ := g.Objects() - Expect(resources).ToNot(BeNil()) - - clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(ok).To(BeTrue()) - - foundUserImp, foundGroupImp, foundSaImp := false, false, false - for _, rule := range clusterRole.Rules { - if rule.Verbs[0] == "impersonate" { - if rule.Resources[0] == "users" { - Expect(rule.ResourceNames).To(Equal([]string{})) - foundUserImp = true - } - if rule.Resources[0] == "groups" { - Expect(rule.ResourceNames).To(Equal([]string{})) - foundGroupImp = true - } - if rule.Resources[0] == "serviceaccounts" { - Expect(rule.ResourceNames).To(Equal([]string{})) - foundSaImp = true - } - } - } - - Expect(foundUserImp).To(BeTrue()) - Expect(foundGroupImp).To(BeTrue()) - Expect(foundSaImp).To(BeTrue()) - }) - - It("should render guardian with specific impersonation", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ - Spec: operatorv1.ManagementClusterConnectionSpec{ - Impersonation: &operatorv1.Impersonation{ - Users: []string{"foo"}, - Groups: []string{"bar"}, - ServiceAccounts: []string{"zaz"}, - }, - }, - } - - g := render.Guardian(cfg) - resources, _ := g.Objects() - Expect(resources).ToNot(BeNil()) - - clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(ok).To(BeTrue()) - - foundUserImp, foundGroupImp, foundSaImp := false, false, false - for _, rule := range clusterRole.Rules { - if rule.Verbs[0] == "impersonate" { - if rule.Resources[0] == "users" { - Expect(rule.ResourceNames).To(Equal([]string{"foo"})) - foundUserImp = true - } - if rule.Resources[0] == "groups" { - Expect(rule.ResourceNames).To(Equal([]string{"bar"})) - foundGroupImp = true - } - if rule.Resources[0] == "serviceaccounts" { - Expect(rule.ResourceNames).To(Equal([]string{"zaz"})) - foundSaImp = true - } - } - } - - Expect(foundUserImp).To(BeTrue()) - Expect(foundGroupImp).To(BeTrue()) - Expect(foundSaImp).To(BeTrue()) - }) - - It("should render guardian with specific no sa permissions but with user and group", func() { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{ - Spec: operatorv1.ManagementClusterConnectionSpec{ - Impersonation: &operatorv1.Impersonation{ - Users: []string{}, - Groups: []string{}, - }, - }, - } - - g := render.Guardian(cfg) - resources, _ := g.Objects() - Expect(resources).ToNot(BeNil()) - - clusterRole, ok := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(ok).To(BeTrue()) - - foundUserImp, foundGroupImp, foundSaImp := false, false, false - for _, rule := range clusterRole.Rules { - if rule.Verbs[0] == "impersonate" { - if rule.Resources[0] == "users" { - foundUserImp = true - } - if rule.Resources[0] == "groups" { - foundGroupImp = true - } - if rule.Resources[0] == "serviceaccounts" { - foundSaImp = true - } - } - } - - Expect(foundUserImp).To(BeTrue()) - Expect(foundGroupImp).To(BeTrue()) - Expect(foundSaImp).To(BeFalse()) - }) - }) - - It("should render SecurityContextConstrains properly when provider is OpenShift", func() { - cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.OpenShift = true - component := render.Guardian(cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - role := rtest.GetResource(resources, render.GuardianClusterRoleName, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(role.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"nonroot-v2"}, - })) - }) - +var _ = Describe("Guardian OSS rendering tests", func() { Context("GuardianPolicy component", func() { - guardianPolicy := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/guardian.json") - guardianPolicyForOCP := testutils.GetExpectedPolicyFromFile("./testutils/expected_policies/guardian_ocp.json") - - renderGuardianPolicy := func(addr string, openshift bool, variant operatorv1.ProductVariant, includeEgressNetworkPolicy bool) { - installation := operatorv1.InstallationSpec{ - Registry: "my-reg/", - } - cfg := createGuardianConfig(installation, addr, openshift) - cfg.Installation.Variant = variant - cfg.IncludeEgressNetworkPolicy = includeEgressNetworkPolicy + It("should render OSS network policy regardless of IncludeEgressNetworkPolicy flag", func() { + // OSS variant should always render a network policy, even when IncludeEgressNetworkPolicy is false + cfg := newGuardianConfig("127.0.0.1:1234") + cfg.Installation.Variant = operatorv1.Calico + cfg.IncludeEgressNetworkPolicy = false g, err := render.GuardianPolicy(cfg) Expect(err).NotTo(HaveOccurred()) - resources, _ = g.Objects() - } - - Context("policy rendering based on variant and IncludeEgressNetworkPolicy", func() { - It("should render OSS network policy regardless of IncludeEgressNetworkPolicy flag", func() { - // OSS variant should always render a network policy, even when IncludeEgressNetworkPolicy is false - renderGuardianPolicy("127.0.0.1:1234", false, operatorv1.Calico, false) - - policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).NotTo(BeNil(), "OSS variant should always render a network policy") - - // The OSS policy has both Ingress and Egress, ending in a Pass so the - // tunnel to the management cluster isn't dropped by the default-deny. - Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) - Expect(policy.Spec.Egress).NotTo(BeEmpty()) - Expect(policy.Spec.Egress[len(policy.Spec.Egress)-1].Action).To(Equal(v3.Pass)) - - // OSS can't express domain-based egress rules. - for _, rule := range policy.Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty()) - } - }) - - It("should render Enterprise network policy without domain-based egress when IncludeEgressNetworkPolicy is false", func() { - // Enterprise variant with IncludeEgressNetworkPolicy=false should render a policy but skip domain-based egress rules - renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, false) - - policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).NotTo(BeNil(), "Enterprise variant should always render a network policy when tier exists") - - // Verify it's the Enterprise policy (should have both Ingress and Egress types) - Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) - Expect(policy.Spec.Egress).NotTo(BeEmpty()) - - // Verify no domain-based egress rules are present - for _, rule := range policy.Spec.Egress { - Expect(rule.Destination.Domains).To(BeEmpty(), - "Domain-based egress rules should not be present when IncludeEgressNetworkPolicy is false") - } - }) - - It("should render Enterprise network policy with domain-based egress when IncludeEgressNetworkPolicy is true", func() { - // Enterprise variant with IncludeEgressNetworkPolicy=true should render the full policy including domain-based egress - renderGuardianPolicy("my-management.example.com:1234", false, operatorv1.CalicoEnterprise, true) - - policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - Expect(policy).NotTo(BeNil(), "Enterprise variant with IncludeEgressNetworkPolicy=true should render a network policy") - - // Verify it's the Enterprise policy (should have both Ingress and Egress types) - Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) - Expect(policy.Spec.Egress).NotTo(BeEmpty()) - - // Verify domain-based egress rule is present - hasDomainRule := false - for _, rule := range policy.Spec.Egress { - if len(rule.Destination.Domains) > 0 { - hasDomainRule = true - break - } - } - Expect(hasDomainRule).To(BeTrue(), "Domain-based egress rule should be present when IncludeEgressNetworkPolicy is true") - }) - }) + resources, _ := g.Objects() - Context("calico-system rendering", func() { policyName := types.NamespacedName{Name: "calico-system.guardian-access", Namespace: "calico-system"} - - getExpectedPolicy := func(name types.NamespacedName, scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { - if name.Name == "calico-system.guardian-access" && scenario.ManagedCluster { - return testutils.SelectPolicyByProvider(scenario, guardianPolicy, guardianPolicyForOCP) - } - - return nil + policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) + Expect(policy).NotTo(BeNil(), "OSS variant should always render a network policy") + + // The OSS policy has both Ingress and Egress, ending in a Pass so the + // tunnel to the management cluster isn't dropped by the default-deny. + Expect(policy.Spec.Types).To(ConsistOf(v3.PolicyTypeIngress, v3.PolicyTypeEgress)) + Expect(policy.Spec.Egress).NotTo(BeEmpty()) + Expect(policy.Spec.Egress[len(policy.Spec.Egress)-1].Action).To(Equal(v3.Pass)) + + // OSS can't express domain-based egress rules. + for _, rule := range policy.Spec.Egress { + Expect(rule.Destination.Domains).To(BeEmpty()) } - - DescribeTable("should render calico-system policy", - func(scenario testutils.CalicoSystemScenario) { - renderGuardianPolicy("127.0.0.1:1234", scenario.OpenShift, operatorv1.CalicoEnterprise, true) - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - expectedPolicy := getExpectedPolicy(policyName, scenario) - Expect(policy).To(Equal(expectedPolicy)) - }, - Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), - Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), - ) - - // The test matrix above validates against an IP-based management cluster address. - // Validate policy adaptation for domain-based management cluster address here. - It("should adapt Guardian policy if ManagementClusterAddr is domain-based", func() { - renderGuardianPolicy("mydomain.io:8080", false, operatorv1.CalicoEnterprise, true) - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - managementClusterEgressRule := policy.Spec.Egress[5] - Expect(managementClusterEgressRule.Destination.Domains).To(Equal([]string{"mydomain.io"})) - Expect(managementClusterEgressRule.Destination.Ports).To(Equal(networkpolicy.Ports(8080))) - }) }) }) }) @@ -448,8 +125,7 @@ var _ = Describe("guardian", func() { } }) It("should render when disabled", func() { - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -459,8 +135,7 @@ var _ = Describe("guardian", func() { It("should render when set to disabled", func() { cfg.TunnelCAType = operatorv1.CATypeTigera - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -471,8 +146,7 @@ var _ = Describe("guardian", func() { It("should render when enabled", func() { cfg.TunnelCAType = operatorv1.CATypePublic - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) @@ -510,8 +184,7 @@ var _ = Describe("guardian", func() { }, } - g := render.Guardian(cfg) - resources, _ := g.Objects() + resources := guardianObjects(cfg) Expect(resources).ToNot(BeNil()) deployment, ok := rtest.GetResource(resources, render.GuardianDeploymentName, render.GuardianNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) diff --git a/pkg/render/inputs.go b/pkg/render/inputs.go new file mode 100644 index 0000000000..019bed13ff --- /dev/null +++ b/pkg/render/inputs.go @@ -0,0 +1,49 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render + +import ( + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/tls/certificatemanagement" +) + +// Inputs is the raw cluster state a controller gathered, carried into render +// modifiers. Only modifiers read it; core operator code never does. +// +// Per-component config is not carried here. A modifier is handed the same typed +// config the core operator rendered the component from. +type Inputs struct { + Installation *operatorv1.InstallationSpec + FelixConfiguration *v3.FelixConfiguration + ClusterDomain string + + // TrustedBundle is the shared CA bundle for the calico-system namespace. + TrustedBundle certificatemanagement.TrustedBundle + + // Extension is opaque data the controller extension produced, usually an artifact + // that can only be created controller-side because it has cluster side effects + // (a keypair, say). Where a controller needs to read it back, the payload is a + // render type so it does not depend on the extension. Nil when none is active. + Extension any +} + +// ExtractExtensionData returns the Extension slot asserted to T, or the zero value +// of T when it is empty or holds a different type. +func ExtractExtensionData[T any](ri Inputs) T { + data, _ := ri.Extension.(T) + return data +} diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index f8f722a033..561099fbe8 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -16,7 +16,6 @@ package kubecontrollers import ( "fmt" - "path/filepath" "slices" "strconv" "strings" @@ -35,19 +34,15 @@ import ( "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rcomp "github.com/tigera/operator/pkg/render/common/components" - relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" - "github.com/tigera/operator/pkg/render/common/rbacmanagement" "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" - "github.com/tigera/operator/pkg/render/monitor" "github.com/tigera/operator/pkg/tls/certificatemanagement" - "github.com/tigera/operator/pkg/url" ) const ( @@ -58,33 +53,12 @@ const ( KubeControllerMetrics = "calico-kube-controllers-metrics" KubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "kube-controller-access" - // WASMPullSecretName is the dedicated image-pull Secret (a renamed copy of - // the install pull secret) that the WAF reconciler replicates into tenant - // namespaces for the Coraza wasm OCI pull. A dedicated name avoids clashing - // with the operator-managed tigera-pull-secret the GatewayAPI render also - // copies into those namespaces (EV-6386). - WASMPullSecretName = "tigera-waf-pull-secret" - - // WASMCACertName is the dedicated CA-bundle ConfigMap (in the controller - // namespace) the WAF reconciler replicates into tenant namespaces for the - // Coraza wasm OCI registry TLS check — a dedicated name avoids clashing with - // the operator-managed tigera-ca-bundle ConfigMap the GatewayAPI render also - // copies there (EV-6386). The source copy is a renamed copy of the trusted - // bundle, provisioned by the core controller and passed in as WASMCACert. - WASMCACertName = "tigera-waf-ca-bundle" - - EsKubeController = "es-calico-kube-controllers" - EsKubeControllerRole = "es-calico-kube-controllers" - EsKubeControllerRoleBinding = "es-calico-kube-controllers" - EsKubeControllerMetrics = "es-calico-kube-controllers-metrics" - EsKubeControllerNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "es-kube-controller-access" + // ManagedClustersWatchRoleBindingName binds kube-controllers to the managed-cluster + // watch ClusterRole. Used by both calico-kube-controllers (in a management cluster) + // and the enterprise es-calico-kube-controllers, so the binding stays generic here. ManagedClustersWatchRoleBindingName = "es-calico-kube-controllers-managed-cluster-watch" - ElasticsearchKubeControllersUserSecret = "tigera-ee-kube-controllers-elasticsearch-access" - ElasticsearchKubeControllersUserName = "tigera-ee-kube-controllers" - ElasticsearchKubeControllersSecureUserSecret = "tigera-ee-kube-controllers-elasticsearch-access-gateway" - ElasticsearchKubeControllersVerificationUserSecret = "tigera-ee-kube-controllers-gateway-verification-credentials" - KubeControllerPrometheusTLSSecret = "calico-kube-controllers-metrics-tls" + KubeControllerPrometheusTLSSecret = "calico-kube-controllers-metrics-tls" // KubeControllersHealthPort is the port the kube-controllers HealthAggregator listens on when run from the // combined calico binary. The legacy per-component image uses file-based health checks instead. @@ -95,13 +69,17 @@ type KubeControllersConfiguration struct { K8sServiceEp k8sapi.ServiceEndpoint K8sServiceEpPodNetwork k8sapi.ServiceEndpoint - Installation *operatorv1.InstallationSpec + Installation *operatorv1.InstallationSpec + Authentication *operatorv1.Authentication + + // ManagementCluster and ManagementClusterConnection are inputs for the enterprise + // es-kube-controllers assembler. No base rendering reads them. ManagementCluster *operatorv1.ManagementCluster ManagementClusterConnection *operatorv1.ManagementClusterConnection - Authentication *operatorv1.Authentication - // Whether or not the LogStorage CRD is present in the cluster. - LogStorageExists bool + // ManagedClusterWatchBinding binds kube-controllers to the managed-cluster watch + // ClusterRole. The assemblers set it; multi-cluster management is not a core feature. + ManagedClusterWatchBinding bool ClusterDomain string MetricsPort int @@ -113,19 +91,18 @@ type KubeControllersConfiguration struct { // namespace to be returned by the rendered. Expected that the calling code // take care to pass the same secret on each reconcile where possible. KubeControllersGatewaySecret *corev1.Secret - WASMPullSecret *corev1.Secret - WASMCACert *corev1.ConfigMap TrustedBundle certificatemanagement.TrustedBundleRO - // Calico Cloud additions. TenantID is only set by the cloud-gated controller path; when empty - // (regular Calico/Calico Enterprise) no cloud env is emitted. + // TenantID is the Calico Cloud tenant. Only the enterprise assembler consumes it. TenantID string - // Cloud indicates kube-controllers is being rendered for a Calico Cloud install. When false the - // cloud-specific RBAC below is not granted and enterprise RBAC is unchanged. + // Cloud reports whether this is a Calico Cloud install. Only the enterprise + // es-kube-controllers assembler consumes it. Cloud bool - MetricsServerTLS certificatemanagement.KeyPairInterface + // ImageOverrides lets a variant swap the kube-controllers image. The controller + // wires in the operator's image overrides; nil resolves to the core image. + ImageOverrides *imageoverride.Overrides // Namespace to be installed into. Namespace string @@ -137,170 +114,115 @@ type KubeControllersConfiguration struct { // If this is nil, then we should run in zero-tenant mode. Tenant *operatorv1.Tenant - // WAFGatewayExtensionEnabled gates the ACTIVE WAF v3 (Gateway API add-on) - // surface on calico-kube-controllers: the WASM_IMAGE / WASM_PULL_SECRET / - // WASM_CA_CERT env vars, the in-process admission webhook, and the gateway - // envoy-proxy wasm image resolution. Sourced from - // `GatewayAPI.spec.extensions.waf.state == Enabled` (default off). - // See design `tigera/designs#25` (PMREQ-384). - WAFGatewayExtensionEnabled bool - - // GatewayAPIPresent is true when the GatewayAPI CR exists (regardless of - // waf.state), so the operator manages the Gateway API + Envoy Gateway CRDs the - // WAF reconcilers watch. It gates the applicationlayer controller enablement, - // its WAF / Gateway-API / EnvoyExtensionPolicy / event / secret RBAC, and the - // WAF_GATEWAY_EXTENSION_ENABLED signal env — a superset of - // WAFGatewayExtensionEnabled. The WAF controller stays wired (and keeps its - // envoyextensionpolicies delete RBAC) while WAF is disabled precisely so it can - // tear down the EnvoyExtensionPolicies it generated, instead of being removed - // in the same reconcile that disables WAF and never getting the chance - // (EV-6751). It de-programs on WAF_GATEWAY_EXTENSION_ENABLED=false. - GatewayAPIPresent bool - - // WAFWebhookServerTLS is the serving certificate for the in-process WAF - // SecLang validating admission webhook hosted by calico-kube-controllers. - // When set (WAF enabled), it is mounted into the Pod and the webhook server - // reads it from WAF_WEBHOOK_CERT_DIR. Issued for the tigera-waf-webhook - // Service DNS name. Nil leaves the Deployment untouched (and the in-process - // server self-disables when the cert is absent). - WAFWebhookServerTLS certificatemanagement.KeyPairInterface - - // WAFWebhookCABundle is the PEM of the CA that issued WAFWebhookServerTLS - // (the operator CA), stamped into the ValidatingWebhookConfiguration's - // caBundle so the apiserver can verify the in-process webhook endpoint. - // Only consulted when WAFGatewayExtensionEnabled is true. - WAFWebhookCABundle []byte - - // RBACManagementEnabled reports whether to render the RBAC management UI access. - // The controller has already applied the variant, the admin's gate and tenancy. - RBACManagementEnabled bool + // The fields below parameterize the generic kube-controllers component. The + // variant assemblers (NewCalicoKubeControllers, the enterprise es builder) + // fill them; the component renders them without any variant or component-name + // branching. + + // Name is the deployment / pod / container name (and the value the metrics + // Service selects on). + Name string + // ConfigName is the KUBE_CONTROLLERS_CONFIG_NAME the binary reconciles. + ConfigName string + // RoleName / RoleBindingName / MetricsName name the ClusterRole, its binding, + // and the Prometheus metrics Service. + RoleName string + RoleBindingName string + MetricsName string + // EnabledControllers is the ENABLED_CONTROLLERS list. The deployment is only + // rendered when it is non-empty. + EnabledControllers []string + // Rules are the ClusterRole policy rules. + Rules []rbacv1.PolicyRule + // NetworkPolicy, when set, is rendered into the install namespace (and the + // deprecated allow-tigera policy named DeprecatedNetworkPolicyName is deleted). + NetworkPolicy *v3.NetworkPolicy + DeprecatedNetworkPolicyName string + // ExtraEnv is appended to the deployment's container env. + ExtraEnv []corev1.EnvVar + // DisableConfigAPI sets DISABLE_KUBE_CONTROLLERS_CONFIG_API. + DisableConfigAPI bool } -func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { - toCreate := []client.Object{kubeControllersCalicoSystemPolicy(cfg)} +// The calico-kube-controllers components expose an extension point. The +// es-calico-kube-controllers deployment shares the underlying type but not these +// wrappers, so a variant never sees it. +type ( + CalicoComponent interface { + render.Component + KubeControllersConfig() *KubeControllersConfiguration + } - if defaultDeny != nil { - toCreate = append(toCreate, defaultDeny) + CalicoPolicyComponent interface { + render.Component + KubeControllersPolicyConfig() *KubeControllersConfiguration } +) - return render.NewPassthrough( - toCreate, - []client.Object{ - // allow-tigera Tier was renamed to calico-system - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("kube-controller-access", cfg.Namespace), - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", common.CalicoNamespace), - }, - ) +// calicoKubeControllers marks the calico-kube-controllers deployment as extendable. +type calicoKubeControllers struct { + render.Component + + cfg *KubeControllersConfiguration } -func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { - kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg) - enabledControllers := []string{"node", "loadbalancer"} - if cfg.Installation.Variant.IsEnterprise() { - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"remoteclusterconfigurations"}, - Verbs: []string{"watch", "list", "get"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "update", "delete"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"usage.tigera.io"}, - Resources: []string{"licenseusagereports"}, - Verbs: []string{"create", "update", "delete", "watch", "list", "get"}, - }, - ) - enabledControllers = append(enabledControllers, "service", "federatedservices", "usage") - // Wire the applicationlayer WAF controller whenever Gateway API is present, - // not only when WAF is enabled, so it stays running (and can tear down its - // generated EnvoyExtensionPolicies) when WAF is disabled. It de-programs vs - // programs based on WAF_GATEWAY_EXTENSION_ENABLED (EV-6751). - if cfg.GatewayAPIPresent { - enabledControllers = append(enabledControllers, "applicationlayer") - } +func (c calicoKubeControllers) KubeControllersConfig() *KubeControllersConfiguration { + return c.cfg +} - // Reconciles ClusterRoles and bindings against the tigera-idp-groups ConfigMap. - if cfg.RBACManagementEnabled { - enabledControllers = append(enabledControllers, "rbacsync") - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, rbacSyncControllerRules()...) - } - } +// calicoKubeControllersPolicy marks the calico-kube-controllers network policy as +// extendable. +type calicoKubeControllersPolicy struct { + render.Component - return &kubeControllersComponent{ - cfg: cfg, - kubeControllerServiceAccountName: KubeControllerServiceAccount, - kubeControllerRoleName: KubeControllerRole, - kubeControllerRoleBindingName: KubeControllerRoleBinding, - kubeControllerName: KubeController, - kubeControllerConfigName: "default", - kubeControllerMetricsName: KubeControllerMetrics, - kubeControllersRules: kubeControllerRolePolicyRules, - enabledControllers: enabledControllers, - } + cfg *KubeControllersConfiguration } -func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent { - var kubeControllerCalicoSystemPolicy *v3.NetworkPolicy - kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg) +func (c calicoKubeControllersPolicy) KubeControllersPolicyConfig() *KubeControllersConfiguration { + return c.cfg +} - if cfg.Installation.Variant.IsEnterprise() { - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) +func NewCalicoKubeControllersPolicy(cfg *KubeControllersConfiguration, defaultDeny *v3.NetworkPolicy) render.Component { + toCreate := []client.Object{kubeControllersCalicoSystemPolicy(cfg)} - // Calico Cloud's es-kube-controllers provisions RBAC for managed-cluster access, so it needs - // to create/update cluster roles and bindings. Enterprise only needs read access. - clusterRoleVerbs := []string{"watch", "list", "get"} - if cfg.Cloud { - clusterRoleVerbs = append(clusterRoleVerbs, "create", "update") - } + if defaultDeny != nil { + toCreate = append(toCreate, defaultDeny) + } - kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, - rbacv1.PolicyRule{ - APIGroups: []string{"elasticsearch.k8s.elastic.co"}, - Resources: []string{"elasticsearches"}, - Verbs: []string{"watch", "get", "list"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings"}, - Verbs: clusterRoleVerbs, + return calicoKubeControllersPolicy{ + Component: render.NewPassthrough( + toCreate, + []client.Object{ + // allow-tigera Tier was renamed to calico-system + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("kube-controller-access", cfg.Namespace), + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", common.CalicoNamespace), }, - ) - - kubeControllerCalicoSystemPolicy = esKubeControllersCalicoSystemPolicy(cfg) + ), + cfg: cfg, } +} - var enabledControllers []string - if !cfg.Tenant.MultiTenant() { - // Zero and single tenant cluster needs elasticsearch configuration - enabledControllers = append(enabledControllers, "authorization", "elasticsearchconfiguration") - if cfg.ManagementCluster != nil && cfg.Tenant == nil { - // Enterprise will require the managedcluster controller to push licenses - enabledControllers = append(enabledControllers, "managedcluster") - } - } +// NewKubeControllers builds a kube-controllers component from a fully-populated +// configuration. Callers (NewCalicoKubeControllers, the enterprise es-kube-controllers +// builder) fill the generic Name/Rules/EnabledControllers/ExtraEnv/NetworkPolicy fields; +// the component renders them with no variant branching. +func NewKubeControllers(cfg *KubeControllersConfiguration) render.Component { + return &kubeControllersComponent{cfg: cfg} +} - return &kubeControllersComponent{ - cfg: cfg, - kubeControllerServiceAccountName: KubeControllerServiceAccount, - kubeControllerRoleName: EsKubeControllerRole, - kubeControllerRoleBindingName: EsKubeControllerRoleBinding, - kubeControllerName: EsKubeController, - kubeControllerConfigName: "elasticsearch", - kubeControllerMetricsName: EsKubeControllerMetrics, - kubeControllersRules: kubeControllerRolePolicyRules, - kubeControllerCalicoSystemPolicy: kubeControllerCalicoSystemPolicy, - enabledControllers: enabledControllers, - } +// NewCalicoKubeControllers builds the calico-kube-controllers component. The base is +// pure OSS; a variant layers its additions on through the installation extension. +func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) render.Component { + cfg.Name = KubeController + cfg.ConfigName = "default" + cfg.RoleName = KubeControllerRole + cfg.RoleBindingName = KubeControllerRoleBinding + cfg.MetricsName = KubeControllerMetrics + + cfg.Rules = KubeControllersRoleCommonRules(cfg) + cfg.EnabledControllers = []string{"node", "loadbalancer"} + + return calicoKubeControllers{Component: NewKubeControllers(cfg), cfg: cfg} } type kubeControllersComponent struct { @@ -309,24 +231,6 @@ type kubeControllersComponent struct { // Internal state generated by the given configuration. calicoImage string - - kubeControllerServiceAccountName string - kubeControllerRoleName string - kubeControllerRoleBindingName string - kubeControllerName string - kubeControllerConfigName string - kubeControllerMetricsName string - - kubeControllersRules []rbacv1.PolicyRule - kubeControllerCalicoSystemPolicy *v3.NetworkPolicy - - enabledControllers []string - - // wasmImage is the fully-resolved OCI reference for the Coraza WAF wasm - // binary (Enterprise only). Surfaced to the kube-controllers binary via - // the WASM_IMAGE env var; consumed by the applicationlayer reconcilers - // in tigera/calico-private to program WAF policy attachments. - wasmImage string } func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error { @@ -334,26 +238,11 @@ func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error path := c.cfg.Installation.ImagePath prefix := c.cfg.Installation.ImagePrefix var err error - if c.cfg.Cloud { - // Calico Cloud runs kube-controllers from the tesla-compiled variant of the combined image, - // which carries the Cloud behavior the enterprise mono image lacks. It is the same binary, - // so the container command and health probes below are unchanged. See TSLA-11580. - c.calicoImage, err = components.GetReference(components.CalicoCloudImage(), reg, path, prefix, is) - } else { - c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) - } + image := c.cfg.ImageOverrides.Resolve(render.ComponentNameKubeControllers, components.CombinedCalicoImage(c.cfg.Installation), c.cfg.Installation) + c.calicoImage, err = components.GetReference(image, reg, path, prefix, is) if err != nil { return err } - if c.cfg.Installation.Variant.IsEnterprise() && c.cfg.WAFGatewayExtensionEnabled { - // The Coraza WAF wasm is baked into the gateway envoy-proxy image as its - // final layer; Envoy Gateway extracts it from there. Point WASM_IMAGE at - // that same image (no standalone coraza-wasm image needed). - c.wasmImage, err = components.GetReference(components.ComponentGatewayAPIEnvoyProxy, reg, path, prefix, is) - if err != nil { - return err - } - } return nil } @@ -365,12 +254,14 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) objectsToCreate := []client.Object{} objectsToDelete := []client.Object{} - if c.kubeControllerCalicoSystemPolicy != nil { - objectsToCreate = append(objectsToCreate, c.kubeControllerCalicoSystemPolicy) - // allow-tigera Tier was renamed to calico-system - objectsToDelete = append(objectsToDelete, - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("es-kube-controller-access", c.cfg.Namespace), - ) + if c.cfg.NetworkPolicy != nil { + objectsToCreate = append(objectsToCreate, c.cfg.NetworkPolicy) + if c.cfg.DeprecatedNetworkPolicyName != "" { + // allow-tigera Tier was renamed to calico-system + objectsToDelete = append(objectsToDelete, + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject(c.cfg.DeprecatedNetworkPolicyName, c.cfg.Namespace), + ) + } } objectsToCreate = append(objectsToCreate, @@ -379,11 +270,8 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) c.controllersClusterRoleBinding(), ) objectsToCreate = append(objectsToCreate, c.managedClusterRoleBindings()...) - if c.kubeControllerName == KubeController && c.cfg.RBACManagementEnabled { - objectsToCreate = append(objectsToCreate, c.rbacSyncNamespacedRole()...) - } - if len(c.enabledControllers) > 0 { + if len(c.cfg.EnabledControllers) > 0 { // There's something to run, so create the deployment. objectsToCreate = append(objectsToCreate, c.controllersDeployment()) } else { @@ -398,26 +286,6 @@ func (c *kubeControllersComponent) Objects() ([]client.Object, []client.Object) objectsToCreate = append(objectsToCreate, secret.ToRuntimeObjects( secret.CopyToNamespace(c.cfg.Namespace, c.cfg.KubeControllersGatewaySecret)...)...) } - if c.cfg.WASMPullSecret != nil { - objectsToCreate = append(objectsToCreate, secret.ToRuntimeObjects( - secret.CopyToNamespace(c.cfg.Namespace, c.cfg.WASMPullSecret)...)...) - } - if c.cfg.WASMCACert != nil { - objectsToCreate = append(objectsToCreate, c.cfg.WASMCACert) - } - - // The in-process WAF admission webhook surface (Service fronting this Pod + - // ValidatingWebhookConfiguration). Rendered here, rather than as a - // passthrough in the core controller, so the objects are cleaned up when the - // WAF extension is disabled or the GatewayAPI CR is removed. - if c.kubeControllerName == KubeController { - webhookObjs := applicationlayer.WAFAdmissionWebhookComponents(c.cfg.WAFWebhookCABundle) - if c.cfg.WAFGatewayExtensionEnabled { - objectsToCreate = append(objectsToCreate, webhookObjs...) - } else { - objectsToDelete = append(objectsToDelete, webhookObjs...) - } - } if c.cfg.MetricsPort != 0 { objectsToCreate = append(objectsToCreate, c.prometheusService()) @@ -437,7 +305,7 @@ func (c *kubeControllersComponent) Ready() bool { return true } -func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { +func KubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { // Nodes are watched to monitor for deletions. @@ -565,400 +433,11 @@ func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration) []rbacv1. return rules } -func kubeControllersRoleEnterpriseCommonRules(cfg *KubeControllersConfiguration) []rbacv1.PolicyRule { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - }, - { - // The Federated Services Controller needs access to the remote kubeconfig secret - // in order to create a remote syncer. - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - }, - { - // Needed to validate the license - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - // Needed to update the status of the LicenseKey with the result of license validation. - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"licensekeys/status"}, - Verbs: []string{"update"}, - }, - { - // The node controller watches Networks so that it can discount an L2 - // network's subnet edges and gateways from the reserved-IP metric. - // The Network resource is only available in Enterprise / Cloud at - // this time. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"list", "watch"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections"}, - Verbs: []string{"get", "watch", "list"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"deeppacketinspections/status"}, - Verbs: []string{"update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures"}, - Verbs: []string{"get", "list", "update"}, - }, - { - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"packetcaptures/status"}, - Verbs: []string{"update"}, - }, - } - - if cfg.GatewayAPIPresent { - // WAF v3 (Gateway API add-on) RBAC. Gated by GatewayAPIPresent, not - // waf.state==Enabled, so the applicationlayer controller keeps the RBAC it - // needs to watch targets and DELETE the EnvoyExtensionPolicies it generated - // while WAF is disabled (EV-6751). The rule set is identical enabled vs - // disabled, so toggling waf.state causes no ClusterRole churn. - rules = append(rules, - // Application-layer (gateway-addons) reconcilers reconcile WAF resources - // against Gateway API targetRefs and emit events on the policy objects. - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - }, - rbacv1.PolicyRule{ - // Validate Gateway API targetRefs and surface attachment status. - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - }, - // controller-runtime Reconcilers (e.g. the applicationlayer manager) record - // events on watched objects via Recorder.Eventf; both core and events.k8s.io - // API groups are emitted depending on the kubernetes version. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - rbacv1.PolicyRule{ - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - }, - // Application-layer reconciler replicates the WAF wasm pull Secret from - // the controller namespace (calico-system) into each WAFPolicy's - // namespace so the rendered EnvoyExtensionPolicy can reference it. Also - // replicates CA-cert ConfigMaps when WASM_CA_CERT is set. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler emits one EnvoyExtensionPolicy per WAF - // targetRef to bind the Coraza wasm filter at the gateway / route. - rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Application-layer reconciler stamps each namespace with its - // allocated WAF rule-id range (applicationlayer.projectcalico.org/waf-id-range - // annotation) so application operators can author in-range rules. The - // base role already grants namespaces get/list/watch; the annotation - // write needs patch/update, gated to the WAF path. - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - }, - ) - } - - if cfg.ManagementClusterConnection != nil { - rules = append(rules, - rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"licensekeys"}, - Verbs: []string{"get", "create", "update", "list", "watch"}, - }, - ) - } - - return rules -} - -// rbacSyncNamespacedRole returns the Role + RoleBinding granting rbacsync read access to -// the two ConfigMaps in calico-system it depends on: tigera-idp-groups and the gate. -func (c *kubeControllersComponent) rbacSyncNamespacedRole() []client.Object { - name := "calico-kube-controllers-rbac-sync" - return []client.Object{ - &rbacv1.Role{ - TypeMeta: metav1.TypeMeta{Kind: "Role", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: common.CalicoNamespace}, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{rbacmanagement.GroupsConfigMapName}, - Verbs: []string{"get", "list", "watch"}, - }, - { - // This cluster's copy of the gate; a managed cluster's is read over - // that cluster's own client. - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{rbacmanagement.ConfigMapName}, - Verbs: []string{"get", "list", "watch"}, - }, - }, - }, - &rbacv1.RoleBinding{ - TypeMeta: metav1.TypeMeta{Kind: "RoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: common.CalicoNamespace}, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: name, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: c.kubeControllerServiceAccountName, - Namespace: c.cfg.Namespace, - }, - }, - }, - } -} - -// rbacSyncControllerRules returns the cluster-scoped rules the rbacsync -// controller holds. The controller reconciles the ClusterRoles that back the -// Manager UI's RBAC management feature, and each rule below lets it manage the -// access one Calico Enterprise UI feature (and its view or modify state) -// requires. The controller runs only when RBAC management is enabled. -// -// Under Kubernetes' privilege-escalation guard the controller can only grant -// permissions it already holds, so each rule mirrors a grant made by one of the -// managed calico-ui-* ClusterRoles the rbacsync controller generates in -// calico-private (kube-controllers/pkg/controllers/rbacsync: resourceroles.go -// defines the calico-ui--{view,mod} and calico-ui-logs-view-* roles, -// tierroles.go the calico-ui-{np,gnp}-{view,mod}- and calico-ui-cluster- -// context roles). The comment on each rule names the managed role(s) it covers. -// -// Only the grants unique to the managed roles live here. Core resources those -// roles also grant (namespaces, nodes, services, pods, clusterinformations, -// hostendpoints, serviceaccounts, tiers) are already held by the common -// kube-controllers rules above, which satisfy the escalation guard for them. -func rbacSyncControllerRules() []rbacv1.PolicyRule { - return []rbacv1.PolicyRule{ - // RBAC management: the ClusterRoles and bindings the controller - // reconciles for the feature. Not a mirrored grant — this is the - // controller's own reconcile target for every managed calico-ui-* role. - { - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles", "clusterrolebindings", "rolebindings"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Network Policy tiers, view and modify: the per-tier and all-tiers - // Policies and Global Policies roles cover the tiers and tier-scoped - // (tier.*) policy resources. The plain networkpolicies and - // stagednetworkpolicies come from Policy Recommendations, which - // references them directly. Mirrors calico-ui-{np,gnp}-{view,mod}- - // (and -all), calico-ui-get-tier-* and calico-ui-policy-recommendations- - // {view,mod}. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "tiers", - "tier.networkpolicies", - "tier.stagednetworkpolicies", - "tier.globalnetworkpolicies", - "tier.stagedglobalnetworkpolicies", - "stagedkubernetesnetworkpolicies", - "networkpolicies", - "stagednetworkpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Network Policy tiers, view and modify: Kubernetes network policies - // within a tier. Mirrors calico-ui-np-{view,mod}- (and -all). - { - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // The per-feature pages, view and modify: Dashboards, Managed Clusters, - // Global Network Sets, Network Sets, Policy Recommendations, Packet - // Captures, Alerts and Security Events, Threat Feeds, Compliance - // Reports, Webhooks, Deep Packet Inspection, and Egress Gateways. - // Mirrors the matching calico-ui--{view,mod} roles - // (e.g. calico-ui-managed-clusters-{view,mod}, calico-ui-alerts- - // {view,mod}, calico-ui-egress-gateways-{view,mod}). - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{ - "uisettings", - "uisettingsgroups", - "globalnetworksets", - "networksets", - "managedclusters", - "policyrecommendationscopes", - "policyrecommendationscopes/status", - "deeppacketinspections", - "deeppacketinspections/status", - "egressgatewaypolicies", - "externalnetworks", - "globalalerts", - "globalalerts/status", - "globalalerttemplates", - "alertexceptions", - "globalthreatfeeds", - "globalthreatfeeds/status", - "globalreports", - "globalreports/status", - "globalreporttypes", - "packetcaptures", - "packetcaptures/files", - "securityeventwebhooks", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Dashboards, view and modify: the cluster-settings and user-settings - // dashboard layouts stored on the UISettingsGroups data subresource. - // Mirrors calico-ui-dashboards-{view,mod}. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - // Manager UI load: the authorization self-check the UI runs on load. - // Packet Captures: authenticating a capture-file download. Mirrors the - // authorizationreviews grant on calico-ui-cluster-context and the - // authenticationreviews grant on calico-ui-packet-captures-{view,mod}. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"authorizationreviews", "authenticationreviews"}, - Verbs: []string{"create"}, - }, - // Manager UI load: Felix configuration read for cluster-wide settings. - // Mirrors calico-ui-cluster-context. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"felixconfigurations"}, - Verbs: []string{"get", "list", "watch"}, - }, - // Webhooks, modify: creating and updating the Secret that stores the - // webhook credentials. Mirrors calico-ui-webhooks-mod. - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"create"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"secrets"}, - ResourceNames: []string{"webhooks-secret"}, - Verbs: []string{"patch"}, - }, - // Logs, view: Flow, DNS, Audit, L7, and Events log access, per managed - // cluster and for the management cluster. Mirrors calico-ui-logs-view-* - // (all/audit/dns/events/flows/l7, plus their per-cluster and - // all-clusters variants). - { - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"cluster"}, - Verbs: []string{"get"}, - }, - // Manager UI load: the Compliance feature-enabled check. Mirrors the - // unscoped compliances grant on calico-ui-cluster-context. (The - // calico-ui-compliance-reports-{view,mod} roles also read compliances - // but scope it to the tigera-secure CR; this rule must stay unscoped to - // cover cluster-context, whose feature check is not resource-scoped.) - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"compliances"}, - Verbs: []string{"get"}, - }, - // Manager UI load: feature-enabled checks for Application Layer / WAF, - // Packet Capture, and Intrusion Detection. Mirrors calico-ui-cluster- - // context (which bundles the compliances check above into the same rule). - { - APIGroups: []string{"operator.tigera.io"}, - Resources: []string{"applicationlayers", "packetcaptureapis", "intrusiondetections"}, - Verbs: []string{"get"}, - }, - // Global Network Sets and Network Sets, view and modify: listing the - // pods a network set selects. Mirrors the pods grant on calico-ui- - // {global-network-sets,network-sets}-{view,mod} and calico-ui-service- - // graph-{view,mod}. (The common kube-controllers rules above already - // grant pods get/list/watch for IPAM GC, so this is also covered there.) - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"list"}, - }, - // Service Graph: the service accounts the flow view references. The only - // managed role granting serviceaccounts, so mirrors calico-ui-service- - // graph-{view,mod}. (That role also grants services, namespaces and - // hostendpoints, all covered by the common kube-controllers rules above.) - { - APIGroups: []string{""}, - Resources: []string{"serviceaccounts"}, - Verbs: []string{"get", "list"}, - }, - // Manager UI load: the statistics proxy to the Calico API server and - // the node Prometheus. Mirrors calico-ui-cluster-context. - { - APIGroups: []string{""}, - Resources: []string{"services/proxy"}, - ResourceNames: []string{"https:calico-api:8080", "calico-node-prometheus:9090"}, - Verbs: []string{"get", "create"}, - }, - } -} - func (c *kubeControllersComponent) controllersServiceAccount() *corev1.ServiceAccount { return &corev1.ServiceAccount{ TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerServiceAccountName, + Name: KubeControllerServiceAccount, Namespace: c.cfg.Namespace, Labels: map[string]string{}, }, @@ -969,9 +448,9 @@ func (c *kubeControllersComponent) controllersClusterRole() *rbacv1.ClusterRole role := &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerRoleName, + Name: c.cfg.RoleName, }, - Rules: c.kubeControllersRules, + Rules: c.cfg.Rules, } return role @@ -994,7 +473,7 @@ func (c *kubeControllersComponent) controllersOCPFederationRoleBinding() *rbacv1 Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: KubeController, + Name: KubeControllerServiceAccount, Namespace: c.cfg.Namespace, }, }, @@ -1003,108 +482,20 @@ func (c *kubeControllersComponent) controllersOCPFederationRoleBinding() *rbacv1 func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { env := []corev1.EnvVar{ - {Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: c.kubeControllerConfigName}, + {Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: c.cfg.ConfigName}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "ENABLED_CONTROLLERS", Value: strings.Join(c.enabledControllers, ",")}, - {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.Tenant.MultiTenant() && c.kubeControllerConfigName == "elasticsearch")}, - } - - if c.cfg.TenantID != "" { - env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.TenantID}) + {Name: "ENABLED_CONTROLLERS", Value: strings.Join(c.cfg.EnabledControllers, ",")}, + {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.DisableConfigAPI)}, } env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...) + env = append(env, c.cfg.ExtraEnv...) - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Tenant != nil { - env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.Tenant.Spec.ID}) - } - - if c.kubeControllerName == EsKubeController { - // What started as a workaround is now the default behaviour. This feature uses our backend in order to - // log into Kibana for users from external identity providers, rather than configuring an authn realm - // in the Elastic stack. - env = append(env, corev1.EnvVar{Name: "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND", Value: "true"}) - - if c.cfg.Authentication != nil { - env = append(env, - corev1.EnvVar{Name: "OIDC_AUTH_USERNAME_PREFIX", Value: c.cfg.Authentication.Spec.UsernamePrefix}, - corev1.EnvVar{Name: "OIDC_AUTH_GROUP_PREFIX", Value: c.cfg.Authentication.Spec.GroupsPrefix}, - ) - } - } - if c.cfg.TrustedBundle != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_CLUSTER_FORWARDING_CA", Value: c.cfg.TrustedBundle.MountPath()}) - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - env = append(env, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - // The WAF reconcilers are wired whenever Gateway API is present (see the - // applicationlayer entry in enabledControllers), so they can tear down the - // EnvoyExtensionPolicies they generated when WAF is disabled. - // WAF_GATEWAY_EXTENSION_ENABLED tells them whether to program (enabled) or - // de-program (disabled) — EV-6751. Absent ⇒ the reconciler defaults to - // enabled, so an older operator that predates this var is unaffected. - if c.cfg.GatewayAPIPresent { - env = append(env, corev1.EnvVar{Name: "WAF_GATEWAY_EXTENSION_ENABLED", Value: strconv.FormatBool(c.cfg.WAFGatewayExtensionEnabled)}) - } - - // Application-layer (gateway-addons / WAF v3) WASM env vars, gated by - // GatewayAPI.spec.extensions.waf.state == Enabled. When the gate is - // off (default), none of the WASM_* env vars are rendered and the - // WAF reconcilers de-program rather than attach a filter. - if c.cfg.WAFGatewayExtensionEnabled { - // Application-layer (gateway-addons) reconcilers consume the Coraza WAF - // wasm OCI reference from this env var to program WAF policy attachments. - // Empty when ResolveImages was not called for the Calico variant; the - // reconciler stamps Programmed=False/WASMUnavailable in that case. - if c.wasmImage != "" { - env = append(env, corev1.EnvVar{Name: "WASM_IMAGE", Value: c.wasmImage}) - } - - // WASM_PULL_SECRET names the imagePullSecret the reconciler replicates - // from the kube-controllers namespace into a WAFPolicy's namespace so - // the rendered EnvoyExtensionPolicy can pull the wasm OCI artifact from - // a private Tigera registry. Source the name from the first - // Installation.ImagePullSecrets entry so multi-tenant / BYO-registry - // installs reuse whatever pull secret operator already attaches here. - if c.cfg.WASMPullSecret != nil { - env = append(env, corev1.EnvVar{Name: "WASM_PULL_SECRET", Value: c.cfg.WASMPullSecret.Name}) - } - - // WASM_CA_CERT names the dedicated CA bundle ConfigMap (provisioned as - // WASMCACert) that the reconciler replicates alongside WASM_PULL_SECRET - // so the EnvoyExtensionPolicy wasm fetcher trusts the registry's TLS - // chain. Only set when the source ConfigMap is actually rendered. - if c.cfg.WASMCACert != nil { - env = append(env, corev1.EnvVar{Name: "WASM_CA_CERT", Value: c.cfg.WASMCACert.Name}) - } - } - } - - if c.cfg.MetricsServerTLS != nil { - env = append(env, - corev1.EnvVar{Name: "TLS_KEY_PATH", Value: c.cfg.MetricsServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "TLS_CRT_PATH", Value: c.cfg.MetricsServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "CLIENT_COMMON_NAME", Value: monitor.PrometheusClientTLSSecretName}, - ) - } if c.cfg.TrustedBundle != nil { env = append(env, corev1.EnvVar{Name: "CA_CRT_PATH", Value: c.cfg.TrustedBundle.MountPath()}, ) } - if c.cfg.WAFWebhookServerTLS != nil { - // The in-process WAF admission webhook server (calico-private - // applicationlayer manager) reads its serving cert (tls.crt/tls.key) - // from this directory; the controller-runtime webhook server only - // registers when the cert is present. - env = append(env, - corev1.EnvVar{Name: "WAF_WEBHOOK_CERT_DIR", Value: filepath.Dir(c.cfg.WAFWebhookServerTLS.VolumeMountCertificateFilePath())}, - ) - } // UID 999 is used in kube-controller Dockerfile. sc := securitycontext.NewNonRootContext() @@ -1137,7 +528,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { } container := corev1.Container{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Image: c.calicoImage, Command: containerCommand, Env: env, @@ -1148,34 +539,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { VolumeMounts: c.kubeControllersVolumeMounts(), } - if c.cfg.WAFWebhookServerTLS != nil { - // Expose the in-process WAF admission-webhook port that the - // tigera-waf-webhook Service forwards to. - container.Ports = append(container.Ports, corev1.ContainerPort{ - Name: "waf-webhook", - ContainerPort: applicationlayer.WAFWebhookContainerPort, - Protocol: corev1.ProtocolTCP, - }) - } - - if c.kubeControllerName == EsKubeController && !c.cfg.Tenant.MultiTenant() { - _, esHost, esPort, _ := url.ParseEndpoint(relasticsearch.GatewayEndpoint(c.SupportedOSType(), c.cfg.ClusterDomain, render.ElasticsearchNamespace)) - container.Env = append(container.Env, []corev1.EnvVar{ - relasticsearch.ElasticHostEnvVar(esHost), - relasticsearch.ElasticPortEnvVar(esPort), - relasticsearch.ElasticUsernameEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticPasswordEnvVar(ElasticsearchKubeControllersUserSecret), - relasticsearch.ElasticCAEnvVar(c.SupportedOSType()), - }...) - } - var initContainers []corev1.Container - if c.cfg.MetricsServerTLS != nil && c.cfg.MetricsServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.MetricsServerTLS.InitContainer(c.cfg.Namespace, sc)) - } - if c.cfg.WAFWebhookServerTLS != nil && c.cfg.WAFWebhookServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.WAFWebhookServerTLS.InitContainer(c.cfg.Namespace, sc)) - } tolerations := appendUniqueTolerations(c.cfg.Installation.ControlPlaneTolerations, rmeta.TolerateCriticalAddonsAndControlPlane...) if c.cfg.Installation.KubernetesProvider.IsGKE() { tolerations = appendUniqueTolerations(tolerations, rmeta.TolerateGKEARM64NoSchedule) @@ -1184,7 +548,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { NodeSelector: c.cfg.Installation.ControlPlaneNodeSelector, Tolerations: tolerations, ImagePullSecrets: c.cfg.Installation.ImagePullSecrets, - ServiceAccountName: c.kubeControllerServiceAccountName, + ServiceAccountName: KubeControllerServiceAccount, InitContainers: initContainers, Containers: []corev1.Container{container}, Volumes: c.kubeControllersVolumes(), @@ -1195,7 +559,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { d := appsv1.Deployment{ TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Namespace: c.cfg.Namespace, }, Spec: appsv1.DeploymentSpec{ @@ -1205,7 +569,7 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerName, + Name: c.cfg.Name, Namespace: c.cfg.Namespace, Annotations: c.annotations(), }, @@ -1237,29 +601,29 @@ func (c *kubeControllersComponent) controllersClusterRoleBinding() *rbacv1.Clust for _, ns := range c.cfg.BindingNamespaces { subjects = append(subjects, rbacv1.Subject{ Kind: "ServiceAccount", - Name: c.kubeControllerServiceAccountName, + Name: KubeControllerServiceAccount, Namespace: ns, }) } return &rbacv1.ClusterRoleBinding{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerRoleBindingName, + Name: c.cfg.RoleBindingName, Labels: map[string]string{}, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", - Name: c.kubeControllerRoleName, + Name: c.cfg.RoleName, }, Subjects: subjects, } } func (c *kubeControllersComponent) managedClusterRoleBindings() []client.Object { - if c.cfg.ManagementCluster != nil { + if c.cfg.ManagedClusterWatchBinding { return []client.Object{ - rcomp.ClusterRoleBinding(ManagedClustersWatchRoleBindingName, render.ManagedClustersWatchClusterRoleName, c.kubeControllerServiceAccountName, []string{c.cfg.Namespace}), + rcomp.ClusterRoleBinding(ManagedClustersWatchRoleBindingName, render.ManagedClustersWatchClusterRoleName, KubeControllerServiceAccount, []string{c.cfg.Namespace}), } } return []client.Object{} @@ -1271,16 +635,16 @@ func (c *kubeControllersComponent) prometheusService() *corev1.Service { return &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: c.kubeControllerMetricsName, + Name: c.cfg.MetricsName, Namespace: c.cfg.Namespace, Annotations: map[string]string{ "prometheus.io/scrape": "true", "prometheus.io/port": fmt.Sprintf("%d", c.cfg.MetricsPort), }, - Labels: map[string]string{"k8s-app": c.kubeControllerName}, + Labels: map[string]string{"k8s-app": c.cfg.Name}, }, Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": c.kubeControllerName}, + Selector: map[string]string{"k8s-app": c.cfg.Name}, // "Headless" service; prevent kube-proxy from rendering any rules for this service // (which is only intended for Prometheus to scrape). ClusterIP: "None", @@ -1309,9 +673,6 @@ func (c *kubeControllersComponent) annotations() map[string]string { am = make(map[string]string) } - if c.cfg.MetricsServerTLS != nil { - am[c.cfg.MetricsServerTLS.HashAnnotationKey()] = c.cfg.MetricsServerTLS.HashAnnotationValue() - } if c.cfg.KubeControllersGatewaySecret != nil { am[render.ElasticsearchUserHashAnnotation] = rmeta.AnnotationHash(c.cfg.KubeControllersGatewaySecret.Data) } @@ -1323,12 +684,6 @@ func (c *kubeControllersComponent) kubeControllersVolumeMounts() []corev1.Volume if c.cfg.TrustedBundle != nil { mounts = append(mounts, c.cfg.TrustedBundle.VolumeMounts(c.SupportedOSType())...) } - if c.cfg.MetricsServerTLS != nil { - mounts = append(mounts, c.cfg.MetricsServerTLS.VolumeMount(c.SupportedOSType())) - } - if c.cfg.WAFWebhookServerTLS != nil { - mounts = append(mounts, c.cfg.WAFWebhookServerTLS.VolumeMount(c.SupportedOSType())) - } return mounts } @@ -1337,12 +692,6 @@ func (c *kubeControllersComponent) kubeControllersVolumes() []corev1.Volume { if c.cfg.TrustedBundle != nil { volumes = append(volumes, c.cfg.TrustedBundle.Volume()) } - if c.cfg.MetricsServerTLS != nil { - volumes = append(volumes, c.cfg.MetricsServerTLS.Volume()) - } - if c.cfg.WAFWebhookServerTLS != nil { - volumes = append(volumes, c.cfg.WAFWebhookServerTLS.Volume()) - } return volumes } @@ -1359,20 +708,6 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }, }...) - if cfg.ManagementClusterConnection != nil { - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: render.GuardianEntityRule, - }) - } else { - egressRules = append(egressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.DefaultHelper().ManagerEntityRule(), - }) - } - ingressRules := []v3.Rule{} if cfg.MetricsPort != 0 { ingressRules = append(ingressRules, v3.Rule{ @@ -1385,20 +720,6 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }) } - // Allow the kube-apiserver to reach the in-process WAF admission webhook on - // :9443 (EV-6386). render-v3 wires the webhook Service/config/cert + the - // server, but without this ingress rule the calico-system default-deny drops - // the apiserver→:9443 call and every WAFPolicy/WAFPlugin admission times out. - if cfg.WAFGatewayExtensionEnabled { - ingressRules = append(ingressRules, v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(uint16(applicationlayer.WAFWebhookContainerPort)), - }, - }) - } - if r, err := cfg.K8sServiceEp.DestinationEntityRule(); r != nil && err == nil { egressRules = append(egressRules, v3.Rule{ Action: v3.Allow, @@ -1423,53 +744,3 @@ func kubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.Ne }, } } - -func esKubeControllersCalicoSystemPolicy(cfg *KubeControllersConfiguration) *v3.NetworkPolicy { - if cfg.ManagementClusterConnection != nil { - return nil - } - - egressRules := []v3.Rule{} - egressRules = networkpolicy.AppendDNSEgressRules(egressRules, cfg.Installation.KubernetesProvider.IsOpenShift()) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: networkpolicy.Ports(443, 6443, 12388), - }, - }, - }...) - - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicy.DefaultHelper().ESGatewayEntityRule(), - }, - }...) - - networkpolicyHelper := networkpolicy.Helper(cfg.Tenant.MultiTenant(), cfg.Namespace) - egressRules = append(egressRules, []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: networkpolicyHelper.ManagerEntityRule(), - }, - }...) - - return &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: EsKubeControllerNetworkPolicyName, - Namespace: cfg.Namespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(EsKubeController), - Types: []v3.PolicyType{v3.PolicyTypeEgress}, - Egress: egressRules, - }, - } -} diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index b785b1e9cd..a75d0c3f68 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -16,12 +16,10 @@ package kubecontrollers_test import ( "fmt" - "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -41,16 +39,13 @@ import ( "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" - "github.com/tigera/operator/pkg/render/applicationlayer" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/networkpolicy" - "github.com/tigera/operator/pkg/render/common/rbacmanagement" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/testutils" - "github.com/tigera/operator/pkg/tls" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) var _ = Describe("kube-controllers rendering tests", func() { @@ -61,40 +56,10 @@ var _ = Describe("kube-controllers rendering tests", func() { cli client.Client ) - esEnvs := []corev1.EnvVar{ - {Name: "ELASTIC_HOST", Value: "tigera-secure-es-gateway-http.tigera-elasticsearch.svc"}, - {Name: "ELASTIC_PORT", Value: "9200", ValueFrom: nil}, - { - Name: "ELASTIC_USERNAME", Value: "", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ee-kube-controllers-elasticsearch-access", - }, - Key: "username", - }, - }, - }, - { - Name: "ELASTIC_PASSWORD", Value: "", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ee-kube-controllers-elasticsearch-access", - }, - Key: "password", - }, - }, - }, - {Name: "ELASTIC_CA", Value: certificatemanagement.TrustedCertBundleMountPath}, - } - expectedPolicyForUnmanaged := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers.json") expectedPolicyForUnmanagedOCP := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers_ocp.json") expectedPolicyForManaged := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers_managed.json") expectedPolicyForManagedOCP := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/kubecontrollers_managed_ocp.json") - expectedESPolicy := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/es-kubecontrollers.json") - expectedESPolicyForOpenshift := testutils.GetExpectedPolicyFromFile("../testutils/expected_policies/es-kubecontrollers_ocp.json") BeforeEach(func() { // Initialize a default instance to use. Each test can override this to its @@ -156,9 +121,12 @@ var _ = Describe("kube-controllers rendering tests", func() { })) }) - It("should use the tesla calico image for kube-controllers when Cloud is enabled (TSLA-11580)", func() { + It("resolves the kube-controllers image through the image overrides", func() { instance.Variant = operatorv1.CalicoEnterprise - cfg.Cloud = true + overrides := imageoverride.New() + overrides.Register(operatorv1.CalicoEnterprise, render.ComponentNameKubeControllers, components.CalicoCloudImage()) + cfg.ImageOverrides = overrides + component := kubecontrollers.NewCalicoKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() @@ -264,176 +232,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateCriticalAddonsAndControlPlane)) }) - It("should render all calico kube-controllers resources for a default configuration (standalone) using CalicoEnterprise", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.KubeControllerServiceAccount, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.KubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.KubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.KubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.WASMPullSecretName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.WASMCACertName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: applicationlayer.WAFWebhookServiceName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - {name: "tigera-waf.applicationlayer.projectcalico.org", ns: "", group: "admissionregistration.k8s.io", version: "v1", kind: "ValidatingWebhookConfiguration"}, - {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - instance.Variant = operatorv1.CalicoEnterprise - instance.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "tigera-pull-secret"}} - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - cfg.GatewayAPIPresent = true - cfg.WAFWebhookCABundle = []byte("fake-ca-bundle") - // core_controller provisions a dedicated WAF wasm pull secret (a renamed - // copy of the install pull secret) so the reconciler can replicate it into - // WAFPolicy namespaces without clashing with the operator-managed - // tigera-pull-secret; surface it here so it renders and WASM_PULL_SECRET is set. - cfg.WASMPullSecret = &corev1.Secret{TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.WASMPullSecretName, Namespace: common.CalicoNamespace}} - // Likewise core_controller provisions the dedicated WAF wasm CA-bundle - // ConfigMap (a renamed copy of the trusted bundle); surface it here so it - // renders and WASM_CA_CERT is set. - cfg.WASMCACert = &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.WASMCACertName, Namespace: common.CalicoNamespace}} - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - Expect(dp.Spec.Template.Spec.ImagePullSecrets).To(ContainElement(corev1.LocalObjectReference{Name: "tigera-pull-secret"})) - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - // Application-layer reconcilers consume these env vars to program WAF - // EnvoyExtensionPolicy attachments. - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_IMAGE", Value: "test-reg/tigera/envoy-proxy:" + components.ComponentGatewayAPIEnvoyProxy.Version, - })) - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_PULL_SECRET", Value: kubecontrollers.WASMPullSecretName, - })) - // WASM_CA_CERT names the dedicated WAF trusted-bundle ConfigMap that the - // reconciler replicates into WAFPolicy namespaces (kept separate from the - // operator-managed tigera-ca-bundle the GatewayAPI render also copies there). - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "WASM_CA_CERT", Value: kubecontrollers.WASMCACertName, - })) - - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) - - clusterRole := rtest.GetResource(resources, kubecontrollers.KubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(39), "cluster role should have 39 rules") - - // Application-layer reconciler RBAC: WAF CRDs (resources, /status, /finalizers). - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies", "globalwafpolicies", - "wafplugins", "globalwafplugins", - "wafvalidationpolicies", "globalwafvalidationpolicies", - }, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/status", "globalwafpolicies/status", - "wafplugins/status", "globalwafplugins/status", - "wafvalidationpolicies/status", "globalwafvalidationpolicies/status", - }, - Verbs: []string{"get", "update", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"applicationlayer.projectcalico.org"}, - Resources: []string{ - "wafpolicies/finalizers", "globalwafpolicies/finalizers", - "wafplugins/finalizers", "globalwafplugins/finalizers", - "wafvalidationpolicies/finalizers", "globalwafvalidationpolicies/finalizers", - }, - Verbs: []string{"update"}, - })) - // Gateway API targetRef validation + status patching. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways", "httproutes", "tcproutes", "tlsroutes", "grpcroutes"}, - Verbs: []string{"get", "list", "watch", "update", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.networking.k8s.io"}, - Resources: []string{"gateways/status", "httproutes/status", "tcproutes/status", "tlsroutes/status", "grpcroutes/status"}, - Verbs: []string{"get", "update", "patch"}, - })) - // Recorder.Eventf emits to both core/events and events.k8s.io/events. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - })) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"events.k8s.io"}, - Resources: []string{"events"}, - Verbs: []string{"create", "patch"}, - })) - // Cluster-wide secrets+configmaps CRUD: reconciler replicates pull - // secrets and CA bundles from the controller namespace into target - // WAFPolicy namespaces. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets", "configmaps"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - // EnvoyExtensionPolicy CRUD: reconciler renders one EEP per WAF targetRef. - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - - ms := rtest.GetResource(resources, kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless") - - // The webhook surface is rendered with the operator CA stamped into the - // ValidatingWebhookConfiguration caBundle. - vwc := rtest.GetResource(resources, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration").(*admissionregistrationv1.ValidatingWebhookConfiguration) - Expect(vwc.Webhooks).To(HaveLen(1)) - Expect(vwc.Webhooks[0].ClientConfig.CABundle).To(Equal([]byte("fake-ca-bundle"))) - }) - - It("should delete the WAF admission webhook surface when the WAF Gateway API add-on is disabled", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.WAFGatewayExtensionEnabled = false - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - toCreate, toDelete := component.Objects() - - // Neither webhook object is created... - Expect(rtest.GetResource(toCreate, applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, "", "v1", "Service")).To(BeNil()) - Expect(rtest.GetResource(toCreate, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration")).To(BeNil()) - // ...and both are queued for deletion, so disabling the feature (or - // removing the GatewayAPI CR) cleans up an earlier enabled render. - Expect(rtest.GetResource(toDelete, applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, "", "v1", "Service")).NotTo(BeNil()) - Expect(rtest.GetResource(toDelete, "tigera-waf.applicationlayer.projectcalico.org", "", "admissionregistration.k8s.io", "v1", "ValidatingWebhookConfiguration")).NotTo(BeNil()) - }) - It("should render all calico kube-controllers resources using CalicoEnterprise on Openshift", func() { expectedResources := []struct { name string @@ -462,179 +260,7 @@ var _ = Describe("kube-controllers rendering tests", func() { } }) - Context("RBAC management UI", func() { - BeforeEach(func() { - instance.Variant = operatorv1.CalicoEnterprise - }) - - It("does not enable rbacsync or grant its RBAC while the feature gate is off", func() { - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage", - })) - - Expect(rtest.GetResource(resources, "calico-kube-controllers-rbac-sync", common.CalicoNamespace, "rbac.authorization.k8s.io", "v1", "Role")).To(BeNil()) - }) - - It("enables rbacsync and grants it read access to both ConfigMaps it depends on", func() { - cfg.RBACManagementEnabled = true - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,rbacsync", - })) - - nsRole := rtest.GetResource(resources, "calico-kube-controllers-rbac-sync", common.CalicoNamespace, "rbac.authorization.k8s.io", "v1", "Role").(*rbacv1.Role) - Expect(nsRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{rbacmanagement.GroupsConfigMapName}, - Verbs: []string{"get", "list", "watch"}, - }), "expected read-only access to tigera-idp-groups in calico-system") - Expect(nsRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - ResourceNames: []string{rbacmanagement.ConfigMapName}, - Verbs: []string{"get", "list", "watch"}, - }), "expected read-only access to the feature gate in calico-system") - }) - }) - - It("should render all es-calico-kube-controllers resources for a default configuration (standalone) using CalicoEnterprise when logstorage and secrets exist", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, - {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - cfg.GatewayAPIPresent = true - - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "authorization,elasticsearchconfiguration", - })) - Expect(envs).To(ContainElements(esEnvs)) - - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) - - Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - - clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(37), "cluster role should have 37 rules") - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - })) - }) - - It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.KubeControllerServiceAccount, ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.KubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.KubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.KubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: applicationlayer.WAFWebhookServiceName, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - {name: "tigera-waf.applicationlayer.projectcalico.org", ns: "", group: "admissionregistration.k8s.io", version: "v1", kind: "ValidatingWebhookConfiguration"}, - {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - // Override configuration to match expected Enterprise config. - instance.Variant = operatorv1.CalicoEnterprise - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - cfg.GatewayAPIPresent = true - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", - Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) - - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - }) It("should render all calico-kube-controllers resources for a default configuration using CalicoEnterprise", func() { - var defaultMode int32 = 420 - var kubeControllerTLS certificatemanagement.KeyPairInterface expectedResources := []struct { name string ns string @@ -649,15 +275,14 @@ var _ = Describe("kube-controllers rendering tests", func() { {name: kubecontrollers.KubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, } + // The metrics serving TLS (TLS_KEY_PATH/TLS_CRT_PATH/CLIENT_COMMON_NAME env, + // the keypair volume + mount) is layered on by the enterprise modifier, so + // the base render here carries only the trusted bundle. expectedEnv := []corev1.EnvVar{ - {Name: "TLS_KEY_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.key"}, - {Name: "TLS_CRT_PATH", Value: "/calico-kube-controllers-metrics-tls/tls.crt"}, - {Name: "CLIENT_COMMON_NAME", Value: "calico-node-prometheus-client-tls"}, {Name: "CA_CRT_PATH", Value: "/etc/pki/tls/certs/tigera-ca-bundle.crt"}, } expectedVolumeMounts := []corev1.VolumeMount{ {Name: "tigera-ca-bundle", MountPath: "/etc/pki/tls/certs", ReadOnly: true}, - {Name: "calico-kube-controllers-metrics-tls", MountPath: "/calico-kube-controllers-metrics-tls", ReadOnly: true}, } expectedVolume := []corev1.Volume{ { @@ -668,34 +293,11 @@ var _ = Describe("kube-controllers rendering tests", func() { }, }, }, - { - Name: "calico-kube-controllers-metrics-tls", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "calico-kube-controllers-metrics-tls", - DefaultMode: &defaultMode, - }, - }, - }, } - scheme := runtime.NewScheme() - Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) - cli := ctrlrfake.DefaultFakeClientBuilder(scheme).Build() - - certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - kubeControllerTLS, err = certificateManager.GetOrCreateKeyPair(cli, - kubecontrollers.KubeControllerPrometheusTLSSecret, - common.OperatorNamespace(), - dns.GetServiceDNSNames(kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, dns.DefaultClusterDomain)) - Expect(err).NotTo(HaveOccurred()) - // Override configuration to match expected Enterprise config. instance.Variant = operatorv1.CalicoEnterprise cfg.MetricsPort = 9094 - cfg.MetricsServerTLS = kubeControllerTLS component := kubecontrollers.NewCalicoKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) @@ -715,179 +317,15 @@ var _ = Describe("kube-controllers rendering tests", func() { envs := dp.Spec.Template.Spec.Containers[0].Env Expect(envs).To(ContainElements(expectedEnv)) - Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(2)) + Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1)) Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElements(expectedVolumeMounts)) - Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(2)) + Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1)) Expect(dp.Spec.Template.Spec.Volumes).To(ContainElements(expectedVolume)) Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) }) - It("should mount the WAF admission webhook serving cert and expose its port when WAF is enabled", func() { - certificateManager, err := certificatemanager.Create(cli, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - wafTLS, err := certificateManager.GetOrCreateKeyPair(cli, - applicationlayer.WAFWebhookServerTLSSecretName, - common.OperatorNamespace(), - dns.GetServiceDNSNames(applicationlayer.WAFWebhookServiceName, common.CalicoNamespace, dns.DefaultClusterDomain)) - Expect(err).NotTo(HaveOccurred()) - - instance.Variant = operatorv1.CalicoEnterprise - cfg.WAFGatewayExtensionEnabled = true - cfg.GatewayAPIPresent = true - cfg.WAFWebhookServerTLS = wafTLS - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - c := dp.Spec.Template.Spec.Containers[0] - - // Serving cert is mounted and advertised to the in-process webhook server. - Expect(dp.Spec.Template.Spec.Volumes).To(ContainElement(wafTLS.Volume())) - Expect(c.VolumeMounts).To(ContainElement(wafTLS.VolumeMount(rmeta.OSTypeLinux))) - Expect(c.Env).To(ContainElement(corev1.EnvVar{ - Name: "WAF_WEBHOOK_CERT_DIR", - Value: filepath.Dir(wafTLS.VolumeMountCertificateFilePath()), - })) - - // In-process webhook port exposed for the tigera-waf-webhook Service. - Expect(c.Ports).To(ContainElement(corev1.ContainerPort{ - Name: "waf-webhook", - ContainerPort: int32(9443), - Protocol: corev1.ProtocolTCP, - })) - - // namespaces patch/update RBAC for the waf-id-range annotation. - clusterRole := rtest.GetResource(resources, "calico-kube-controllers", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - Verbs: []string{"get", "patch", "update"}, - })) - }) - - It("should keep the WAF controller wired for teardown but render no active WAF surface when WAF is disabled (EV-6751)", func() { - instance.Variant = operatorv1.CalicoEnterprise - // GatewayAPI present but WAF turned off: the applicationlayer controller - // must stay wired (with its EnvoyExtensionPolicy delete RBAC) so it can - // tear down the EEPs it generated, and be told it is disabled via the env. - cfg.GatewayAPIPresent = true - cfg.WAFGatewayExtensionEnabled = false - - component := kubecontrollers.NewCalicoKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - c := dp.Spec.Template.Spec.Containers[0] - - // Controller stays wired. - Expect(c.Env).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage,applicationlayer", - })) - // Told it is disabled → the reconciler de-programs rather than attaches. - Expect(c.Env).To(ContainElement(corev1.EnvVar{Name: "WAF_GATEWAY_EXTENSION_ENABLED", Value: "false"})) - // No active WAF surface: no WASM image env, no webhook cert dir. - for _, e := range c.Env { - Expect(e.Name).NotTo(Equal("WASM_IMAGE")) - Expect(e.Name).NotTo(Equal("WAF_WEBHOOK_CERT_DIR")) - } - // But it keeps the EnvoyExtensionPolicy delete RBAC to run the teardown. - clusterRole := rtest.GetResource(resources, "calico-kube-controllers", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"gateway.envoyproxy.io"}, - Resources: []string{"envoyextensionpolicies"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - })) - }) - - It("should render all es-calico-kube-controllers resources for a default configuration using CalicoEnterprise and ClusterType is Management", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: kubecontrollers.EsKubeControllerNetworkPolicyName, ns: common.CalicoNamespace, group: "projectcalico.org", version: "v3", kind: "NetworkPolicy"}, - {name: "calico-kube-controllers", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: kubecontrollers.EsKubeControllerRole, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: kubecontrollers.EsKubeControllerRoleBinding, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.ManagedClustersWatchRoleBindingName, ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: kubecontrollers.EsKubeController, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "Deployment"}, - {name: kubecontrollers.ElasticsearchKubeControllersUserSecret, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Secret"}, - {name: kubecontrollers.EsKubeControllerMetrics, ns: common.CalicoNamespace, group: "", version: "v1", kind: "Service"}, - } - - // Override configuration to match expected Enterprise config. - instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - // Opt in to the WAF Gateway API add-on so the WAF env vars + RBAC are rendered. - cfg.WAFGatewayExtensionEnabled = true - cfg.GatewayAPIPresent = true - - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The Deployment should have the correct configuration. - dp := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - - envs := dp.Spec.Template.Spec.Containers[0].Env - Expect(envs).To(ContainElement(corev1.EnvVar{ - Name: "ENABLED_CONTROLLERS", - Value: "authorization,elasticsearchconfiguration,managedcluster", - })) - - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/etc/pki/tls/certs")) - - Expect(dp.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(dp.Spec.Template.Spec.Volumes[0].Name).To(Equal("tigera-ca-bundle")) - Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle")) - - Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.ComponentTigeraCalico.Version)) - - clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(clusterRole.Rules).To(HaveLen(37), "cluster role should have 37 rules") - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"watch", "list", "get", "update", "create", "delete"}, - })) - Expect(clusterRole.Rules).To(ContainElement( - rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"secrets"}, - Verbs: []string{"watch", "list", "get"}, - })) - roleBindingWatch := rtest.GetResource(resources, kubecontrollers.ManagedClustersWatchRoleBindingName, "", "rbac.authorization.k8s.io", "v1", "ClusterRoleBinding").(*rbacv1.ClusterRoleBinding) - Expect(roleBindingWatch.RoleRef.Name).To(Equal(render.ManagedClustersWatchClusterRoleName)) - Expect(roleBindingWatch.Subjects).To(ConsistOf([]rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: kubecontrollers.KubeControllerServiceAccount, - Namespace: common.CalicoNamespace, - }, - })) - }) - It("should include a ControlPlaneNodeSelector when specified", func() { expectedResources := []struct { name string @@ -983,44 +421,6 @@ var _ = Describe("kube-controllers rendering tests", func() { Expect(passed).To(Equal(true)) }) - It("should add the OIDC prefix env variables", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - cfg.Authentication = &operatorv1.Authentication{Spec: operatorv1.AuthenticationSpec{ - UsernamePrefix: "uOIDC:", - GroupsPrefix: "gOIDC:", - Openshift: &operatorv1.AuthenticationOpenshift{IssuerURL: "https://api.example.com"}, - }} - - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - depResource := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") - Expect(depResource).ToNot(BeNil()) - deployment := depResource.(*appsv1.Deployment) - - var usernamePrefix, groupPrefix string - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == kubecontrollers.EsKubeController { - for _, env := range container.Env { - switch env.Name { - case "OIDC_AUTH_USERNAME_PREFIX": - usernamePrefix = env.Value - case "OIDC_AUTH_GROUP_PREFIX": - groupPrefix = env.Value - } - } - } - } - - Expect(usernamePrefix).To(Equal("uOIDC:")) - Expect(groupPrefix).To(Equal("gOIDC:")) - }) - Context("With calico-kube-controllers overrides", func() { rr1 := corev1.ResourceRequirements{ Limits: corev1.ResourceList{ @@ -1251,35 +651,6 @@ var _ = Describe("kube-controllers rendering tests", func() { }) }) - When("enableESOIDCWorkaround is true", func() { - It("should set the ENABLE_ELASTICSEARCH_OIDC_WORKAROUND env variable to true", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true - cfg.ManagementCluster = &operatorv1.ManagementCluster{} - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - cfg.MetricsPort = 9094 - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) - resources, _ := component.Objects() - - depResource := rtest.GetResource(resources, kubecontrollers.EsKubeController, common.CalicoNamespace, "apps", "v1", "Deployment") - Expect(depResource).ToNot(BeNil()) - deployment := depResource.(*appsv1.Deployment) - - var esLicenseType string - for _, container := range deployment.Spec.Template.Spec.Containers { - if container.Name == kubecontrollers.EsKubeController { - for _, env := range container.Env { - if env.Name == "ENABLE_ELASTICSEARCH_OIDC_WORKAROUND" { - esLicenseType = env.Value - } - } - } - } - - Expect(esLicenseType).To(Equal("true")) - }) - }) - It("should add the KUBERNETES_SERVICE_... variables", func() { cfg.K8sServiceEpPodNetwork = k8sapi.ServiceEndpoint{ Host: "k8shost", @@ -1387,70 +758,6 @@ var _ = Describe("kube-controllers rendering tests", func() { }) }) - Context("es-kube-controllers calico-system rendering", func() { - policyName := types.NamespacedName{Name: "calico-system.es-kube-controller-access", Namespace: common.CalicoNamespace} - - getExpectedPolicy := func(scenario testutils.CalicoSystemScenario) *v3.NetworkPolicy { - if scenario.ManagedCluster { - return nil - } - - return testutils.SelectPolicyByProvider(scenario, expectedESPolicy, expectedESPolicyForOpenshift) - } - - DescribeTable("should render calico-system policy", - func(scenario testutils.CalicoSystemScenario) { - if scenario.OpenShift { - cfg.Installation.KubernetesProvider = operatorv1.ProviderOpenShift - } else { - cfg.Installation.KubernetesProvider = operatorv1.ProviderNone - } - if scenario.ManagedCluster { - cfg.ManagementClusterConnection = &operatorv1.ManagementClusterConnection{} - } else { - cfg.ManagementClusterConnection = nil - } - instance.Variant = operatorv1.CalicoEnterprise - cfg.LogStorageExists = true - cfg.KubeControllersGatewaySecret = &testutils.KubeControllersUserSecret - - component := kubecontrollers.NewElasticsearchKubeControllers(&cfg) - resources, _ := component.Objects() - - policy := testutils.GetCalicoSystemPolicyFromResources(policyName, resources) - expectedPolicy := getExpectedPolicy(scenario) - Expect(policy).To(Equal(expectedPolicy)) - }, - Entry("for management/standalone, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: false}), - Entry("for management/standalone, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: false, OpenShift: true}), - Entry("for managed, kube-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: false}), - Entry("for managed, openshift-dns", testutils.CalicoSystemScenario{ManagedCluster: true, OpenShift: true}), - ) - }) - - It("should render init containers when certificate management is enabled", func() { - instance.Variant = operatorv1.CalicoEnterprise - cfg.MetricsPort = 9094 - ca, _ := tls.MakeCA(rmeta.DefaultOperatorCASignerName()) - cert, _, _ := ca.Config.GetPEMBytes() // create a valid pem block - cfg.Installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: cert} - - certificateManager, err := certificatemanager.Create(cli, cfg.Installation, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) - Expect(err).NotTo(HaveOccurred()) - - tls, err := certificateManager.GetOrCreateKeyPair(cli, kubecontrollers.KubeControllerPrometheusTLSSecret, common.OperatorNamespace(), []string{""}) - Expect(err).NotTo(HaveOccurred()) - - cfg.MetricsServerTLS = tls - - resources, _ := kubecontrollers.NewCalicoKubeControllers(&cfg).Objects() - - dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) - Expect(dp.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - csrInitContainer := dp.Spec.Template.Spec.InitContainers[0] - Expect(csrInitContainer.Name).To(Equal(fmt.Sprintf("%v-key-cert-provisioner", kubecontrollers.KubeControllerPrometheusTLSSecret))) - }) - It("should add egress policy with Enterprise variant and K8SServiceEndpoint defined", func() { cfg.K8sServiceEp.Host = "k8shost" cfg.K8sServiceEp.Port = "1234" diff --git a/pkg/render/kubecontrollers/waf_pull_secret.go b/pkg/render/kubecontrollers/waf_pull_secret.go deleted file mode 100644 index 02ada09f8a..0000000000 --- a/pkg/render/kubecontrollers/waf_pull_secret.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright (c) 2026 Tigera, Inc. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kubecontrollers - -import ( - "encoding/json" - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/tigera/operator/pkg/common" -) - -// MergeWAFPullSecret synthesizes the dedicated WAF wasm pull secret -// (tigera-waf-pull-secret) by merging the registry auths of every Installation -// pull secret. The EnvoyExtensionPolicy image source takes a single -// pullSecretRef, so a merged secret is the only way to honor multiple -// Installation pull secrets for the Coraza wasm OCI pull (e.g. the Tigera pull -// secret plus credentials for a private registry mirror). -// -// If the same registry appears in more than one secret, the first secret in -// Installation order wins. Secrets that cannot be parsed are skipped and their -// names returned, so the caller can log them without failing the reconcile. -// Returns a nil Secret when no registry auths could be collected. -func MergeWAFPullSecret(pullSecrets []*corev1.Secret) (*corev1.Secret, []string) { - merged := map[string]json.RawMessage{} - var skipped []string - for _, s := range pullSecrets { - auths, err := registryAuths(s) - if err != nil { - skipped = append(skipped, s.Name) - continue - } - for registry, auth := range auths { - if _, ok := merged[registry]; !ok { - merged[registry] = auth - } - } - } - if len(merged) == 0 { - return nil, skipped - } - - // Marshalling a map sorts its keys, so the rendered bytes are deterministic - // and do not churn the object on every reconcile. - data, err := json.Marshal(map[string]map[string]json.RawMessage{"auths": merged}) - if err != nil { - // Each auth entry round-trips from a successful Unmarshal above, so - // this cannot fail in practice; treat it as nothing to render. - return nil, skipped - } - - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: WASMPullSecretName, Namespace: common.CalicoNamespace}, - Type: corev1.SecretTypeDockerConfigJson, - Data: map[string][]byte{corev1.DockerConfigJsonKey: data}, - }, skipped -} - -// registryAuths extracts the per-registry auth entries from a pull secret of -// either the dockerconfigjson type (auths nested under an "auths" key) or the -// legacy dockercfg type (a bare registry -> auth map). -func registryAuths(s *corev1.Secret) (map[string]json.RawMessage, error) { - if raw, ok := s.Data[corev1.DockerConfigJsonKey]; ok { - var cfg struct { - Auths map[string]json.RawMessage `json:"auths"` - } - if err := json.Unmarshal(raw, &cfg); err != nil { - return nil, err - } - if len(cfg.Auths) == 0 { - return nil, fmt.Errorf("secret %s has no auths entries", s.Name) - } - return cfg.Auths, nil - } - if raw, ok := s.Data[corev1.DockerConfigKey]; ok { - var auths map[string]json.RawMessage - if err := json.Unmarshal(raw, &auths); err != nil { - return nil, err - } - if len(auths) == 0 { - return nil, fmt.Errorf("secret %s has no auths entries", s.Name) - } - return auths, nil - } - return nil, fmt.Errorf("secret %s has neither a %s nor a %s key", s.Name, corev1.DockerConfigJsonKey, corev1.DockerConfigKey) -} diff --git a/pkg/render/logstorage/esgateway/esgateway_test.go b/pkg/render/logstorage/esgateway/esgateway_test.go index 13ae8d5c68..cb0a713e04 100644 --- a/pkg/render/logstorage/esgateway/esgateway_test.go +++ b/pkg/render/logstorage/esgateway/esgateway_test.go @@ -37,12 +37,12 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" "github.com/tigera/operator/pkg/dns" + entkubecontrollers "github.com/tigera/operator/pkg/enterprise/kubecontrollers" "github.com/tigera/operator/pkg/render" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/podaffinity" rtest "github.com/tigera/operator/pkg/render/common/test" - "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/logstorage" "github.com/tigera/operator/pkg/render/testutils" "github.com/tigera/operator/pkg/tls" @@ -82,9 +82,9 @@ var _ = Describe("ES Gateway rendering tests", func() { ESGatewayKeyPair: kp, TrustedBundle: bundle, KubeControllersUserSecrets: []*corev1.Secret{ - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - {ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + {ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + {ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, }, ClusterDomain: clusterDomain, EsAdminUserName: "elastic", @@ -96,9 +96,9 @@ var _ = Describe("ES Gateway rendering tests", func() { It("should render an ES Gateway deployment and all supporting resources", func() { expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, @@ -143,9 +143,9 @@ var _ = Describe("ES Gateway rendering tests", func() { expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, @@ -182,9 +182,9 @@ var _ = Describe("ES Gateway rendering tests", func() { installation.CertificateManagement = &operatorv1.CertificateManagement{CACert: secret.Data[corev1.TLSCertKey]} expectedResources := []client.Object{ &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, - &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: entkubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, diff --git a/pkg/render/manager.go b/pkg/render/manager.go index b8cfbc1c4d..94d14e6ebf 100644 --- a/pkg/render/manager.go +++ b/pkg/render/manager.go @@ -278,10 +278,10 @@ func (c *managerComponent) Objects() ([]client.Object, []client.Object) { // For multi-tenant environments, the management cluster itself isn't shown in the UI so we only need to create these // when there is no tenant. objsToCreate = append(objsToCreate, - managerClusterWideSettingsGroup(), - managerUserSpecificSettingsGroup(), - managerClusterWideTigeraLayer(), - managerClusterWideDefaultView(), + ManagerClusterWideSettingsGroup(), + ManagerUserSpecificSettingsGroup(), + ManagerClusterWideTigeraLayer(), + ManagerClusterWideDefaultView(), ) // Continue to create the legacy namespace so that we can create our external name service that points to the new // manager service. This will help ease transition for customers and avoid outages caused by the name and namespace @@ -1496,10 +1496,10 @@ func (c *managerComponent) multiTenantManagedClustersAccess() []client.Object { return objects } -// managerClusterWideSettingsGroup returns a UISettingsGroup with the description "cluster-wide settings" +// ManagerClusterWideSettingsGroup returns a UISettingsGroup with the description "cluster-wide settings" // // Calico Enterprise only -func managerClusterWideSettingsGroup() *v3.UISettingsGroup { +func ManagerClusterWideSettingsGroup() *v3.UISettingsGroup { return &v3.UISettingsGroup{ TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -1511,10 +1511,10 @@ func managerClusterWideSettingsGroup() *v3.UISettingsGroup { } } -// managerUserSpecificSettingsGroup returns a UISettingsGroup with the description "user settings" +// ManagerUserSpecificSettingsGroup returns a UISettingsGroup with the description "user settings" // // Calico Enterprise only -func managerUserSpecificSettingsGroup() *v3.UISettingsGroup { +func ManagerUserSpecificSettingsGroup() *v3.UISettingsGroup { return &v3.UISettingsGroup{ TypeMeta: metav1.TypeMeta{Kind: "UISettingsGroup", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ @@ -1527,11 +1527,11 @@ func managerUserSpecificSettingsGroup() *v3.UISettingsGroup { } } -// managerClusterWideTigeraLayer returns a UISettings layer belonging to the cluster-wide settings group that contains +// ManagerClusterWideTigeraLayer returns a UISettings layer belonging to the cluster-wide settings group that contains // all of the tigera namespaces. // // Calico Enterprise only -func managerClusterWideTigeraLayer() *v3.UISettings { +func ManagerClusterWideTigeraLayer() *v3.UISettings { namespaces := []string{ "tigera-dex", "tigera-dpi", @@ -1575,11 +1575,11 @@ func managerClusterWideTigeraLayer() *v3.UISettings { } } -// managerClusterWideDefaultView returns a UISettings view belonging to the cluster-wide settings group that shows +// ManagerClusterWideDefaultView returns a UISettings view belonging to the cluster-wide settings group that shows // everything and uses the tigera-infrastructure layer. // // Calico Enterprise only -func managerClusterWideDefaultView() *v3.UISettings { +func ManagerClusterWideDefaultView() *v3.UISettings { return &v3.UISettings{ TypeMeta: metav1.TypeMeta{Kind: "UISettings", APIVersion: "projectcalico.org/v3"}, ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/render/node.go b/pkg/render/node.go index d69376e636..18b037faa7 100644 --- a/pkg/render/node.go +++ b/pkg/render/node.go @@ -35,6 +35,7 @@ import ( "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" "github.com/tigera/operator/pkg/controller/migration" + "github.com/tigera/operator/pkg/imageoverride" rcomp "github.com/tigera/operator/pkg/render/common/components" "github.com/tigera/operator/pkg/render/common/configmap" rmeta "github.com/tigera/operator/pkg/render/common/meta" @@ -68,13 +69,15 @@ const ( CalicoCNIPluginObjectName = "calico-cni-plugin" BPFVolumeName = "bpffs" + InstallCNIContainerName = "install-cni" + goldmaneDomainName = "goldmane.calico-system.svc" ) var ( - // The port used by calico/node to report Calico Enterprise BGP metrics. + // NodeBGPReporterPort is the port used by calico/node to report Calico Enterprise BGP metrics. // This is currently not intended to be user configurable. - nodeBGPReporterPort int32 = 9900 + NodeBGPReporterPort int32 = 9900 NodeTLSSecretName = "node-certs" NodeTLSSecretNameNonClusterHost = NodeTLSSecretName + TyphaNonClusterHostSuffix @@ -115,11 +118,9 @@ type NodeConfiguration struct { GoldmaneIP string // Optional fields. - LogCollector *operatorv1.LogCollector - MigrateNamespaces bool - NodeAppArmorProfile string - BirdTemplates map[string]string - NodeReporterMetricsPort int + MigrateNamespaces bool + NodeAppArmorProfile string + BirdTemplates map[string]string // CanRemoveCNIFinalizer specifies whether CNI plugin is still needed during uninstall since the CNI plugin and // associated RBAC resources are required for pod teardown to succeed. Setting this to true removes @@ -127,8 +128,6 @@ type NodeConfiguration struct { // For details on why this is needed see 'Node and Installation finalizer' in the core_controller. CanRemoveCNIFinalizer bool - PrometheusServerTLS certificatemanagement.KeyPairInterface - // BGPLayouts is returned by the rendering code after modifying its namespace // so that it can be deployed into the cluster. // TODO: The controller should pass the contents, the renderer should build its own @@ -146,11 +145,12 @@ type NodeConfiguration struct { // should this value change. BindMode string - FelixPrometheusMetricsEnabled bool - - FelixPrometheusMetricsPort int - V3CRDs bool + + // ImageOverrides lets a variant swap the node and cni-plugins images. The + // controller wires in the operator's image overrides; nil resolves to the + // core images. + ImageOverrides *imageoverride.Overrides } // Node creates the node daemonset and other resources for the daemonset to operate normally. @@ -185,18 +185,11 @@ func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { } c.calicoImage = appendIfErr(components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is)) + nodeImage := c.cfg.ImageOverrides.Resolve(ComponentNameNode, components.ComponentCalicoNode, c.cfg.Installation) + c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if c.installUpstreamPlugins() { - if c.cfg.Installation.Variant.IsEnterprise() { - c.cniPluginsImage = appendIfErr(components.GetReference(components.ComponentTigeraCNIPlugins, reg, path, prefix, is)) - } else { - c.cniPluginsImage = appendIfErr(components.GetReference(components.ComponentCalicoCNIPlugins, reg, path, prefix, is)) - } - } - switch { - case c.cfg.Installation.Variant.IsEnterprise(): - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNode, reg, path, prefix, is)) - default: - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNode, reg, path, prefix, is)) + cniPluginsImage := c.cfg.ImageOverrides.Resolve(ComponentNameCNIPlugins, components.ComponentCalicoCNIPlugins, c.cfg.Installation) + c.cniPluginsImage = appendIfErr(components.GetReference(cniPluginsImage, reg, path, prefix, is)) } if len(errMsgs) != 0 { @@ -205,6 +198,10 @@ func (c *nodeComponent) ResolveImages(is *operatorv1.ImageSet) error { return nil } +func (c *nodeComponent) NodeConfig() *NodeConfiguration { + return c.cfg +} + func (c *nodeComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } @@ -234,11 +231,6 @@ func (c *nodeComponent) Objects() ([]client.Object, []client.Object) { var objsToDelete []client.Object - if c.cfg.Installation.Variant.IsEnterprise() { - // Include Service for exposing node metrics. - objs = append(objs, c.nodeMetricsService()) - } - cniConfig := c.nodeCNIConfigMap() if cniConfig != nil { objs = append(objs, cniConfig) @@ -567,34 +559,6 @@ func (c *nodeComponent) nodeRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - extraRules := []rbacv1.PolicyRule{ - { - // Calico Enterprise needs to be able to read additional resources. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "bfdconfigurations", - "egressgatewaypolicies", - "externalnetworks", - "licensekeys", - "networks", - "packetcaptures", - "remoteclusterconfigurations", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - // Tigera Secure updates status for packet captures. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "packetcaptures", - "packetcaptures/status", - }, - Verbs: []string{"update"}, - }, - } - role.Rules = append(role.Rules, extraRules...) - } if c.cfg.Installation.KubernetesProvider.IsOpenShift() { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, @@ -656,14 +620,6 @@ func (c *nodeComponent) cniPluginRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - // The Network resource is only available in Enterprise / Cloud at this time. - role.Rules = append(role.Rules, rbacv1.PolicyRule{ - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{"networks"}, - Verbs: []string{"get"}, - }) - } return role } @@ -951,18 +907,11 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo if len(c.cfg.BirdTemplates) != 0 { annotations[birdTemplateHashAnnotation] = rmeta.AnnotationHash(c.cfg.BirdTemplates) } - if c.cfg.PrometheusServerTLS != nil { - annotations[c.cfg.PrometheusServerTLS.HashAnnotationKey()] = c.cfg.PrometheusServerTLS.HashAnnotationValue() - } if c.cfg.TLS.NodeSecret.UseCertificateManagement() { initContainers = append(initContainers, c.cfg.TLS.NodeSecret.InitContainer(common.CalicoNamespace, nodeContainer.SecurityContext)) } - if c.cfg.PrometheusServerTLS != nil && c.cfg.PrometheusServerTLS.UseCertificateManagement() { - initContainers = append(initContainers, c.cfg.PrometheusServerTLS.InitContainer(common.CalicoNamespace, nodeContainer.SecurityContext)) - } - if cniCfgMap != nil { annotations[nodeCniConfigAnnotation] = rmeta.AnnotationHash(cniCfgMap.Data) } @@ -1078,10 +1027,6 @@ func (c *nodeComponent) nodeDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1.Daemo ds.Spec.Template.Spec.InitContainers = append(ds.Spec.Template.Spec.InitContainers, c.cniContainer()) } - if c.collectProcessPathEnabled() { - ds.Spec.Template.Spec.HostPID = true - } - SetNodeCriticalPod(&(ds.Spec.Template)) if c.cfg.MigrateNamespaces { migration.LimitDaemonSetToMigratedNodes(&ds) @@ -1113,16 +1058,18 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { c.cfg.TLS.NodeSecret.Volume(), c.varRunCalicoVolume(), c.varLibCalicoVolume(), + // The Calico log directory, which the CNI plugin logs into and Felix may write logs into. + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, // Volume for the containing directory so that the init container can mount the child bpf directory if needed. - corev1.Volume{Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, + {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, // Volume for the bpffs itself, used by the main node container. - corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, + {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, // securityfs, read by Felix to detect kernel lockdown=confidentiality. No // Type set (like nodeproc) so nodes without securityfs still start; Felix // treats an unreadable lockdown file as "not locked down". - corev1.Volume{Name: "sys-kernel-security", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/kernel/security"}}}, + {Name: "sys-kernel-security", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/kernel/security"}}}, // Volume used by mount-cgroupv2 init container to access root cgroup name space of node. - corev1.Volume{Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, + {Name: "nodeproc", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/proc"}}}, } if c.vppDataplaneEnabled() { @@ -1136,7 +1083,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { volumes = append(volumes, corev1.Volume{Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.BinDir, Type: &dirOrCreate}}}) volumes = append(volumes, corev1.Volume{Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: *c.cfg.Installation.CNI.ConfDir}}}) - volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}) } if c.installUpstreamPlugins() { // Staging volume populated by the cni-plugins init container and read @@ -1144,16 +1090,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) } - // Override with Tigera-specific config. - if c.cfg.Installation.Variant.IsEnterprise() { - // Add volume for calico logs. - calicoLogVol := corev1.Volume{ - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, - } - volumes = append(volumes, calicoLogVol) - } - // Create and append flexvolume if c.cfg.Installation.FlexVolumePath != "None" { volumes = append(volumes, corev1.Volume{ @@ -1190,10 +1126,6 @@ func (c *nodeComponent) nodeVolumes() []corev1.Volume { }, }) } - if c.cfg.PrometheusServerTLS != nil { - volumes = append(volumes, c.cfg.PrometheusServerTLS.Volume()) - } - return volumes } @@ -1237,12 +1169,6 @@ func (c *nodeComponent) vppDataplaneEnabled() bool { *c.cfg.Installation.CalicoNetwork.LinuxDataplane == operatorv1.LinuxDataplaneVPP } -func (c *nodeComponent) collectProcessPathEnabled() bool { - return c.cfg.LogCollector != nil && - c.cfg.LogCollector.Spec.CollectProcessPath != nil && - *c.cfg.LogCollector.Spec.CollectProcessPath == operatorv1.CollectProcessPathEnable -} - // cniContainer creates the node's init container that installs CNI. func (c *nodeComponent) cniContainer() corev1.Container { // Determine environment to pass to the CNI init container. @@ -1258,7 +1184,7 @@ func (c *nodeComponent) cniContainer() corev1.Container { } return corev1.Container{ - Name: "install-cni", + Name: InstallCNIContainerName, Image: c.calicoImage, Command: []string{components.CalicoBinaryPath, "component", "cni", "install"}, Env: cniEnv, @@ -1411,12 +1337,6 @@ func (c *nodeComponent) cniEnvvars() []corev1.EnvVar { envVars = append(envVars, c.cfg.K8sServiceEp.EnvVars()...) - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - envVars = append(envVars, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - } - return envVars } @@ -1463,15 +1383,9 @@ func (c *nodeComponent) nodeVolumeMounts() []corev1.VolumeMount { if c.vppDataplaneEnabled() { nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/usr/local/bin/felix-plugins", Name: "felix-plugins", ReadOnly: true}) } - if c.cfg.Installation.Variant.IsEnterprise() { - extraNodeMounts := []corev1.VolumeMount{ - {MountPath: "/var/log/calico", Name: "var-log-calico"}, - } - nodeVolumeMounts = append(nodeVolumeMounts, extraNodeMounts...) - } else if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { - cniLogMount := corev1.VolumeMount{MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false} - nodeVolumeMounts = append(nodeVolumeMounts, cniLogMount) - } + // Mount the Calico log directory. The CNI plugin writes to the cni/ subdirectory + // and, on the enterprise variant, Felix writes its flow/DNS logs here too. + nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico", Name: "var-log-calico"}) if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { nodeVolumeMounts = append(nodeVolumeMounts, corev1.VolumeMount{MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}) @@ -1507,9 +1421,6 @@ func (c *nodeComponent) nodeVolumeMounts() []corev1.VolumeMount { SubPath: BGPLayoutConfigMapKey, }) } - if c.cfg.PrometheusServerTLS != nil { - nodeVolumeMounts = append(nodeVolumeMounts, c.cfg.PrometheusServerTLS.VolumeMount(c.SupportedOSType())) - } return nodeVolumeMounts } @@ -1620,10 +1531,6 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { } } - if c.collectProcessPathEnabled() { - nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_FLOWLOGSCOLLECTPROCESSPATH", Value: "true"}) - } - // Determine MTU to use. If specified explicitly, use that. Otherwise, set defaults based on an overall // MTU of 1460. mtu := getMTU(c.cfg.Installation) @@ -1717,35 +1624,6 @@ func (c *nodeComponent) nodeEnvVars() []corev1.EnvVar { nodeEnv = append(nodeEnv, corev1.EnvVar{Name: "FELIX_IPV6SUPPORT", Value: "false"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add in Calico Enterprise specific configuration. - extraNodeEnv := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", c.cfg.NodeReporterMetricsPort)}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - extraNodeEnv = append(extraNodeEnv, corev1.EnvVar{Name: "MULTI_INTERFACE_MODE", Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value()}) - } - - if c.cfg.PrometheusServerTLS != nil { - extraNodeEnv = append(extraNodeEnv, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: c.cfg.TLS.TrustedBundle.MountPath()}, - ) - } - nodeEnv = append(nodeEnv, extraNodeEnv...) - } - if c.cfg.Installation.NodeMetricsPort != nil { // If a node metrics port was given, then enable felix prometheus metrics and set the port. // Note that this takes precedence over any FelixConfiguration resources in the cluster. @@ -1824,10 +1702,7 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr var readinessCmd []string readinessCmd = []string{components.CalicoBinaryPath, "component", "node", "health", "--bird-ready", "--felix-ready"} - if c.cfg.Installation.Variant.IsEnterprise() { - readinessCmd = append(readinessCmd, "--bgp-metrics-ready") - } - // If not using BGP or using VPP, don't check bird status (or bgp metrics server for enterprise). + // If not using BGP or using VPP, don't check bird status. if !bgpEnabled(c.cfg.Installation) || c.vppDataplaneEnabled() { readinessCmd = []string{components.CalicoBinaryPath, "component", "node", "health", "--felix-ready"} } @@ -1853,56 +1728,6 @@ func (c *nodeComponent) nodeLivenessReadinessProbes() (*corev1.Probe, *corev1.Pr return lp, rp } -// nodeMetricsService creates a Service which exposes two endpoints on calico/node for -// reporting Prometheus metrics (for policy enforcement activity and BGP stats). -// This service is used internally by Calico Enterprise and is separate from general -// Prometheus metrics which are user-configurable. -func (c *nodeComponent) nodeMetricsService() *corev1.Service { - ports := []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(c.cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(c.cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: nodeBGPReporterPort, - TargetPort: intstr.FromInt(int(nodeBGPReporterPort)), - Protocol: corev1.ProtocolTCP, - }, - } - - if c.cfg.FelixPrometheusMetricsEnabled { - felixMetricsPort := int32(c.cfg.FelixPrometheusMetricsPort) - - ports = append(ports, corev1.ServicePort{ - Name: "felix-metrics-port", - Port: felixMetricsPort, - TargetPort: intstr.FromInt(int(felixMetricsPort)), - Protocol: corev1.ProtocolTCP, - }) - } - - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: CalicoNodeMetricsService, - Namespace: common.CalicoNamespace, - Labels: map[string]string{"k8s-app": CalicoNodeObjectName}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": CalicoNodeObjectName}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: ports, - }, - } -} - // getAutodetectionMethod returns the IP auto detection method in a form understandable by the calico/node // startup processing. It returns an empty string if IP auto detection should not be enabled. func getAutodetectionMethod(ad *operatorv1.NodeAddressAutodetection) string { diff --git a/pkg/render/node_test.go b/pkg/render/node_test.go index 0c1c095ee5..2ad6930ef3 100644 --- a/pkg/render/node_test.go +++ b/pkg/render/node_test.go @@ -38,6 +38,7 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -134,14 +135,13 @@ var _ = Describe("Node rendering tests", func() { // Create a default configuration. cfg = render.NodeConfiguration{ - K8sServiceEp: k8sServiceEp, - Installation: defaultInstance, - TLS: typhaNodeTLS, - ClusterDomain: defaultClusterDomain, - FelixHealthPort: 9099, - IPPools: defaultInstance.CalicoNetwork.IPPools, - FelixPrometheusMetricsEnabled: false, - FelixPrometheusMetricsPort: 9098, + K8sServiceEp: k8sServiceEp, + Installation: defaultInstance, + TLS: typhaNodeTLS, + ClusterDomain: defaultClusterDomain, + FelixHealthPort: 9099, + IPPools: defaultInstance.CalicoNetwork.IPPools, + ImageOverrides: imageoverride.New(), } }) @@ -321,7 +321,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -361,7 +361,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, {MountPath: "/sys/kernel/security", Name: "sys-kernel-security", ReadOnly: true}, } @@ -370,7 +370,7 @@ var _ = Describe("Node rendering tests", func() { // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render node correctly for BPF dataplane", func() { @@ -515,7 +515,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, {Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}}, @@ -555,7 +555,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, {MountPath: "/sys/kernel/security", Name: "sys-kernel-security", ReadOnly: true}, } @@ -564,7 +564,7 @@ var _ = Describe("Node rendering tests", func() { // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render a pinned CNI spec version in the CNI config", func() { @@ -671,143 +671,6 @@ var _ = Describe("Node rendering tests", func() { } }) - It("should render all resources for a default configuration using CalicoEnterprise", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - verifyInitContainers(ds, defaultInstance) - - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9099"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - // Expect 2 Ports when FelixPrometheusMetricsEnabled is false - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(len(ms.Spec.Ports)).To(Equal(2)) - - dirMustExist := corev1.HostPathDirectory - bpfVol := corev1.Volume{Name: "bpffs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs/bpf", Type: &dirMustExist}}} - Expect(ds.Spec.Template.Spec.Volumes).To(ContainElement(bpfVol)) - - bpfVolMount := corev1.VolumeMount{MountPath: "/sys/fs/bpf", Name: "bpffs"} - Expect(ds.Spec.Template.Spec.Containers[0].VolumeMounts).To(ContainElement(bpfVolMount)) - - verifyProbesAndLifecycle(ds, false, true) - }) - - It("should render felix service metric with FelixPrometheusMetricPort when FelixPrometheusMetricsEnabled is true", func() { - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixPrometheusMetricsEnabled = true - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - expectedServicePorts := []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: 9900, - TargetPort: intstr.FromInt(int(9900)), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "felix-metrics-port", - Port: 9098, - TargetPort: intstr.FromInt(int(9098)), - Protocol: corev1.ProtocolTCP, - }, - } - - // Expect 3 Ports when FelixPrometheusMetricsEnabled is true - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.Ports).To(Equal(expectedServicePorts)) - }) - It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { name string @@ -945,7 +808,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -985,7 +848,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, {MountPath: "/sys/kernel/security", Name: "sys-kernel-security", ReadOnly: true}, } @@ -996,7 +859,7 @@ var _ = Describe("Node rendering tests", func() { // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render a configuration using the AmazonVPC CNI plugin", func() { @@ -1085,6 +948,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1120,6 +984,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/run/xtables.lock", Name: "xtables-lock"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, @@ -1132,7 +997,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should return customized CNI directories when specified", func() { @@ -1152,7 +1017,7 @@ var _ = Describe("Node rendering tests", func() { expectedVols := []corev1.Volume{ {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/custom/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ContainElements(expectedVols)) @@ -1201,7 +1066,7 @@ var _ = Describe("Node rendering tests", func() { } // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }, Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal, []corev1.EnvVar{ {Name: "FELIX_INTERFACEPREFIX", Value: "gke"}, @@ -1353,7 +1218,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1393,7 +1258,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/sys/fs/bpf", Name: "bpffs"}, {MountPath: "/sys/kernel/security", Name: "sys-kernel-security", ReadOnly: true}, } @@ -1403,7 +1268,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should properly render a configuration using the AmazonVPC CNI plugin", func() { @@ -1490,6 +1355,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "sys-fs", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/sys/fs", Type: &dirOrCreate}}}, @@ -1525,6 +1391,7 @@ var _ = Describe("Node rendering tests", func() { {MountPath: "/run/xtables.lock", Name: "xtables-lock"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "/var/log/calico", Name: "var-log-calico"}, {MountPath: "/var/run/nodeagent", Name: "policysync"}, {MountPath: "/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, @@ -1537,7 +1404,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) It("should render all resources when running on openshift", func() { @@ -1604,7 +1471,7 @@ var _ = Describe("Node rendering tests", func() { {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/cni/bin", Type: &dirOrCreate}}}, {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/multus/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni"}}}, + {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, {Name: "cni-plugins-stage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, {Name: "flexvol-driver-host", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds", Type: &dirOrCreate}}}, @@ -1669,215 +1536,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - verifyProbesAndLifecycle(ds, true, false) - }) - - It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - defaultCNIConfDir, defaultCNIBinDir := render.DefaultCNIDirectories(defaultInstance.KubernetesProvider) - defaultInstance.CNI.ConfDir, defaultInstance.CNI.BinDir = &defaultCNIConfDir, &defaultCNIBinDir - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixHealthPort = 9199 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // calico-node clusterRole should have openshift securitycontextconstraints PolicyRule - nodeRole := rtest.GetResource(resources, "calico-node", "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole) - Expect(nodeRole.Rules).To(ContainElement(rbacv1.PolicyRule{ - APIGroups: []string{"security.openshift.io"}, - Resources: []string{"securitycontextconstraints"}, - Verbs: []string{"use"}, - ResourceNames: []string{"privileged"}, - })) - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - verifyInitContainers(ds, defaultInstance) - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9199"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - verifyProbesAndLifecycle(ds, true, true) - }) - - It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-node", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-cni-plugin", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ServiceAccount"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRole"}, - {name: "calico-cni-plugin", ns: "", group: "rbac.authorization.k8s.io", version: "v1", kind: "ClusterRoleBinding"}, - {name: "calico-node-metrics", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.NodeDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - defaultCNIConfDir, defaultCNIBinDir := render.DefaultCNIDirectories(defaultInstance.KubernetesProvider) - defaultInstance.CNI.ConfDir, defaultInstance.CNI.BinDir = &defaultCNIConfDir, &defaultCNIBinDir - cfg.NodeReporterMetricsPort = 9081 - cfg.FelixHealthPort = 9199 - - component := render.Node(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Image).To(Equal(components.TigeraRegistry + "tigera/node:" + components.ComponentTigeraNode.Version)) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - verifyInitContainers(ds, defaultInstance) - - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "bird"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - {Name: "FELIX_HEALTHPORT", Value: "9199"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "NO_DEFAULT_POOLS", Value: "true"}, - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - - // The RKE2 envvar overrides. - {Name: "MULTI_INTERFACE_MODE", Value: operatorv1.MultiInterfaceModeNone.Value()}, - } - expectedNodeEnv = configureExpectedNodeEnvIPVersions(expectedNodeEnv, defaultInstance, enableIPv4, enableIPv6) - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - - verifyProbesAndLifecycle(ds, true, true) - - // The metrics service should have the correct configuration. - ms := rtest.GetResource(resources, "calico-node-metrics", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") + verifyProbesAndLifecycle(ds, true) }) It("should render volumes and node volumemounts when bird templates are provided", func() { @@ -2136,12 +1795,11 @@ var _ = Describe("Node rendering tests", func() { It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { defaultInstance.Variant = operatorv1.CalicoEnterprise defaultInstance.NodeMetricsPort = nil - cfg.NodeReporterMetricsPort = 9081 component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + Expect(len(resources)).To(Equal(defaultNumExpectedResources)) dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") Expect(dsResource).ToNot(BeNil()) @@ -2150,7 +1808,8 @@ var _ = Describe("Node rendering tests", func() { ds := dsResource.(*appsv1.DaemonSet) Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) - // It should have the reporter port, though. + // The reporter port env is added by the enterprise node modifier, not the + // base render, so it should be absent here. expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) }) @@ -2162,7 +1821,7 @@ var _ = Describe("Node rendering tests", func() { component := render.Node(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) + Expect(len(resources)).To(Equal(defaultNumExpectedResources)) dsResource := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") Expect(dsResource).ToNot(BeNil()) @@ -2990,7 +2649,7 @@ var _ = Describe("Node rendering tests", func() { Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) // Verify readiness and liveness probes. - verifyProbesAndLifecycle(ds, false, false) + verifyProbesAndLifecycle(ds, false) }) DescribeTable("test node probes", @@ -3015,7 +2674,7 @@ var _ = Describe("Node rendering tests", func() { Expect(dsResource).ToNot(BeNil()) ds := dsResource.(*appsv1.DaemonSet) - verifyProbesAndLifecycle(ds, isOpenshift, isEnterprise) + verifyProbesAndLifecycle(ds, isOpenshift) }, Entry("k8s Calico OS no BGP", false, false, operatorv1.BGPDisabled), @@ -3336,7 +2995,7 @@ var _ = Describe("Node rendering tests", func() { }) // verifyProbesAndLifecycle asserts the expected node liveness and readiness probe plus pod lifecycle settings. -func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift, isEnterprise bool) { +func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift bool) { // Verify readiness and liveness probes. expectedReadiness := &corev1.Probe{ PeriodSeconds: 10, @@ -3370,14 +3029,14 @@ func verifyProbesAndLifecycle(ds *appsv1.DaemonSet, isOpenshift, isEnterprise bo } ExpectWithOffset(1, found).To(BeTrue()) + // The base render produces the same readiness command for all variants; the + // enterprise --bgp-metrics-ready check is added by the node modifier and is + // covered in the enterprise package tests. var expectedReadinessCmd []string - switch { - case !bgp: - expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--felix-ready"} - case bgp && isEnterprise: - expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--bird-ready", "--felix-ready", "--bgp-metrics-ready"} - case bgp: + if bgp { expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--bird-ready", "--felix-ready"} + } else { + expectedReadinessCmd = []string{"/usr/bin/calico", "component", "node", "health", "--felix-ready"} } expectedReadiness.ProbeHandler = corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: expectedReadinessCmd}} diff --git a/pkg/render/render_test.go b/pkg/render/render_test.go index 2b97cfaeb9..85ec726dc4 100644 --- a/pkg/render/render_test.go +++ b/pkg/render/render_test.go @@ -66,9 +66,7 @@ func allCalicoComponents( nodeAppArmorProfile string, clusterDomain string, kubeControllersMetricsPort int, - nodeReporterMetricsPort int, bgpLayout *corev1.ConfigMap, - logCollector *operatorv1.LogCollector, ) ([]render.Component, error) { namespaces := render.Namespaces(&render.NamespaceConfiguration{Installation: cr, PullSecrets: pullSecrets}) @@ -79,17 +77,15 @@ func allCalicoComponents( secretsAndConfigMaps := render.NewCreationPassthrough(objs...) nodeCfg := &render.NodeConfiguration{ - K8sServiceEp: k8sServiceEp, - Installation: cr, - TLS: typhaNodeTLS, - NodeAppArmorProfile: nodeAppArmorProfile, - ClusterDomain: clusterDomain, - NodeReporterMetricsPort: nodeReporterMetricsPort, - BGPLayouts: bgpLayout, - LogCollector: logCollector, - BirdTemplates: bt, - MigrateNamespaces: up, - FelixHealthPort: 9099, + K8sServiceEp: k8sServiceEp, + Installation: cr, + TLS: typhaNodeTLS, + NodeAppArmorProfile: nodeAppArmorProfile, + ClusterDomain: clusterDomain, + BGPLayouts: bgpLayout, + BirdTemplates: bt, + MigrateNamespaces: up, + FelixHealthPort: 9099, } typhaCfg := &render.TyphaConfiguration{ K8sServiceEp: k8sServiceEp, @@ -111,13 +107,12 @@ func allCalicoComponents( } winCfg := &render.WindowsConfiguration{ - K8sServiceEp: k8sServiceEp, - K8sDNSServers: []string{}, - Installation: cr, - ClusterDomain: clusterDomain, - TLS: typhaNodeTLS, - NodeReporterMetricsPort: nodeReporterMetricsPort, - VXLANVNI: 4096, + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{}, + Installation: cr, + ClusterDomain: clusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, } nodeCertComponent := rcertificatemanagement.CertificateManagement(&rcertificatemanagement.Config{ @@ -219,22 +214,22 @@ var _ = Describe("Rendering tests", func() { // - 6 kube-controllers resources (ServiceAccount, ClusterRole, Binding, Deployment, Service, Secret,RoleBinding) // - 1 namespace // - 2 Windows node resources (ConfigMap, DaemonSet) - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) It("should render all resources when variant is Tigera Secure", func() { - // For this scenario, we expect the basic resources plus the following for Tigera Secure: - // - X Same as default config - // - 1 Service to expose calico/node metrics. - // - 1 Service to expose Windows calico/node metrics. + // For this scenario, we expect the basic resources plus the following for Tigera Secure. + // The calico/node and Windows calico/node metrics Services are added by the + // enterprise modifiers at the componentHandler, not by Objects(), so they do + // not appear in this render-only aggregation. var nodeMetricsPort int32 = 9081 instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - Expect(componentCount(c)).To(Equal((5 + 3 + 4 + 1 + 6 + 6 + 1 + 2) + 1 + 1)) + Expect(componentCount(c)).To(Equal(5 + 3 + 4 + 1 + 6 + 6 + 1 + 2)) }) It("should render all resources when variant is Tigera Secure and Management Cluster", func() { @@ -245,7 +240,7 @@ var _ = Describe("Rendering tests", func() { instance.Variant = operatorv1.CalicoEnterprise instance.NodeMetricsPort = &nodeMetricsPort - c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, 0, nil, nil) + c, err := allCalicoComponents(k8sServiceEp, instance, &operatorv1.ManagementCluster{}, nil, nil, typhaNodeTLS, internalManagerKeyPair, nil, false, "", dns.DefaultClusterDomain, 9094, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) expectedResources := []client.Object{ @@ -267,7 +262,6 @@ var _ = Describe("Rendering tests", func() { &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "calico-cni-plugin"}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-node-metrics", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cni-config", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: common.NodeDaemonSetName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, @@ -275,12 +269,11 @@ var _ = Describe("Rendering tests", func() { &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllersDeploymentName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ServiceAccount", APIVersion: "v1"}}, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllersDeploymentName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}}, &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllersDeploymentName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, - &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ManagedClustersWatchRoleBindingName}, TypeMeta: metav1.TypeMeta{Kind: "ClusterRoleBinding", APIVersion: "rbac.authorization.k8s.io/v1"}}, &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: common.KubeControllersDeploymentName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "calico-kube-controllers-metrics", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, - // Windows node objects. - &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: render.WindowsNodeMetricsService, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}}, + // Windows node objects. The Windows node-metrics Service is added by the + // enterprise modifier at the componentHandler, so it is not in this output. &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cni-config-windows", Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}}, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: common.WindowsDaemonSetName, Namespace: common.CalicoNamespace}, TypeMeta: metav1.TypeMeta{Kind: "DaemonSet", APIVersion: "apps/v1"}}, @@ -307,7 +300,7 @@ var _ = Describe("Rendering tests", func() { It("should render calico with a apparmor profile if annotation is present in installation", func() { apparmorProf := "foobar" - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, apparmorProf, dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -331,7 +324,7 @@ var _ = Describe("Rendering tests", func() { } bgpLayout.Name = "bgp-layout" bgpLayout.Namespace = common.OperatorNamespace() - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, bgpLayout, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, bgpLayout) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cm *corev1.ConfigMap var ds *appsv1.DaemonSet @@ -352,42 +345,8 @@ var _ = Describe("Rendering tests", func() { Expect(ds.Spec.Template.Annotations["hash.operator.tigera.io/bgp-layout"]).NotTo(BeEmpty()) }) - It("should handle collectProcessPath in logCollector", func() { - testNode := func(processPath operatorv1.CollectProcessPathOption, expectedHostPID bool) { - var logCollector operatorv1.LogCollector - logCollector.Spec.CollectProcessPath = &processPath - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, &logCollector) - Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) - var ds *appsv1.DaemonSet - for _, comp := range comps { - resources, _ := comp.Objects() - r := rtest.GetResource(resources, "calico-node", "calico-system", "apps", "v1", "DaemonSet") - if r != nil { - ds = r.(*appsv1.DaemonSet) - } - } - checkEnvVar := func(ds *appsv1.DaemonSet) bool { - envPresent := false - for _, env := range ds.Spec.Template.Spec.Containers[0].Env { - if env.Name == "FELIX_FLOWLOGSCOLLECTPROCESSPATH" { - envPresent = true - if env.Value == "true" { - return true - } - } - } - return !envPresent - } - Expect(ds).ToNot(BeNil()) - Expect(ds.Spec.Template.Spec.HostPID).To(Equal(expectedHostPID)) - Expect(checkEnvVar(ds)).To(Equal(true)) - } - testNode(operatorv1.CollectProcessPathEnable, true) - testNode(operatorv1.CollectProcessPathDisable, false) - }) - It("should set node priority class to system-node-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.DaemonSet for _, comp := range comps { @@ -403,7 +362,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set typha priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { @@ -419,7 +378,7 @@ var _ = Describe("Rendering tests", func() { }) It("should set kube controllers priority class to system-cluster-critical", func() { - comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, 0, nil, nil) + comps, err := allCalicoComponents(k8sServiceEp, instance, nil, nil, nil, typhaNodeTLS, nil, nil, false, "", dns.DefaultClusterDomain, 0, nil) Expect(err).To(BeNil(), "Expected Calico to create successfully %s", err) var cn *appsv1.Deployment for _, comp := range comps { diff --git a/pkg/render/testutils/expected_policies/kubecontrollers.json b/pkg/render/testutils/expected_policies/kubecontrollers.json index c1770d569f..ab23a78a9e 100644 --- a/pkg/render/testutils/expected_policies/kubecontrollers.json +++ b/pkg/render/testutils/expected_policies/kubecontrollers.json @@ -34,17 +34,6 @@ 12388 ] } - }, - { - "action": "Allow", - "protocol": "TCP", - "destination": { - "selector": "k8s-app == 'calico-manager'", - "namespaceSelector": "kubernetes.io/metadata.name == 'calico-system'", - "ports": [ - 9443 - ] - } } ], "ingress": [ diff --git a/pkg/render/testutils/expected_policies/kubecontrollers_managed.json b/pkg/render/testutils/expected_policies/kubecontrollers_managed.json index 5a41bd4b0c..ab23a78a9e 100644 --- a/pkg/render/testutils/expected_policies/kubecontrollers_managed.json +++ b/pkg/render/testutils/expected_policies/kubecontrollers_managed.json @@ -34,17 +34,6 @@ 12388 ] } - }, - { - "action": "Allow", - "protocol": "TCP", - "destination": { - "selector": "k8s-app == 'guardian'", - "namespaceSelector": "kubernetes.io/metadata.name == 'calico-system'", - "ports": [ - 8080 - ] - } } ], "ingress": [ diff --git a/pkg/render/testutils/expected_policies/kubecontrollers_managed_ocp.json b/pkg/render/testutils/expected_policies/kubecontrollers_managed_ocp.json index 12ddf594c9..b6ed3db73c 100644 --- a/pkg/render/testutils/expected_policies/kubecontrollers_managed_ocp.json +++ b/pkg/render/testutils/expected_policies/kubecontrollers_managed_ocp.json @@ -45,17 +45,6 @@ 12388 ] } - }, - { - "action": "Allow", - "protocol": "TCP", - "destination": { - "selector": "k8s-app == 'guardian'", - "namespaceSelector": "kubernetes.io/metadata.name == 'calico-system'", - "ports": [ - 8080 - ] - } } ], "ingress": [ diff --git a/pkg/render/testutils/expected_policies/kubecontrollers_ocp.json b/pkg/render/testutils/expected_policies/kubecontrollers_ocp.json index 82464e93b4..b6ed3db73c 100644 --- a/pkg/render/testutils/expected_policies/kubecontrollers_ocp.json +++ b/pkg/render/testutils/expected_policies/kubecontrollers_ocp.json @@ -45,17 +45,6 @@ 12388 ] } - }, - { - "action": "Allow", - "protocol": "TCP", - "destination": { - "selector": "k8s-app == 'calico-manager'", - "namespaceSelector": "kubernetes.io/metadata.name == 'calico-system'", - "ports": [ - 9443 - ] - } } ], "ingress": [ diff --git a/pkg/render/typha.go b/pkg/render/typha.go index 264851785f..14cc233318 100644 --- a/pkg/render/typha.go +++ b/pkg/render/typha.go @@ -52,6 +52,8 @@ const ( TyphaContainerName = "calico-typha" + TyphaClusterRoleName = "calico-typha" + TyphaNonClusterHostSuffix = "-noncluster-host" TyphaNonClusterHostNetworkPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "typha-noncluster-host-access" @@ -109,6 +111,10 @@ func (c *typhaComponent) ResolveImages(is *operatorv1.ImageSet) error { return err } +func (c *typhaComponent) TyphaConfig() *TyphaConfiguration { + return c.cfg +} + func (c *typhaComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeLinux } @@ -207,7 +213,7 @@ func (c *typhaComponent) typhaRole() *rbacv1.ClusterRole { role := &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{Kind: "ClusterRole", APIVersion: "rbac.authorization.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ - Name: "calico-typha", + Name: TyphaClusterRoleName, Labels: map[string]string{}, }, @@ -356,26 +362,6 @@ func (c *typhaComponent) typhaRole() *rbacv1.ClusterRole { }, }, } - if c.cfg.Installation.Variant.IsEnterprise() { - extraRules := []rbacv1.PolicyRule{ - { - // Tigera Secure needs to be able to read licenses, and config. - APIGroups: []string{"projectcalico.org", "crd.projectcalico.org"}, - Resources: []string{ - "bfdconfigurations", - "deeppacketinspections", - "egressgatewaypolicies", - "externalnetworks", - "licensekeys", - "networks", - "packetcaptures", - "remoteclusterconfigurations", - }, - Verbs: []string{"get", "list", "watch"}, - }, - } - role.Rules = append(role.Rules, extraRules...) - } if c.cfg.Installation.KubernetesProvider.IsOpenShift() { role.Rules = append(role.Rules, rbacv1.PolicyRule{ APIGroups: []string{"security.openshift.io"}, @@ -633,15 +619,6 @@ func (c *typhaComponent) typhaEnvVars(typhaSecret certificatemanagement.KeyPairI typhaEnv = append(typhaEnv, corev1.EnvVar{Name: "FELIX_INTERFACEPREFIX", Value: "azv"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - if c.cfg.Installation.CalicoNetwork != nil && c.cfg.Installation.CalicoNetwork.MultiInterfaceMode != nil { - typhaEnv = append(typhaEnv, corev1.EnvVar{ - Name: "MULTI_INTERFACE_MODE", - Value: c.cfg.Installation.CalicoNetwork.MultiInterfaceMode.Value(), - }) - } - } - // If host-local IPAM is in use, we need to configure typha to use the Kubernetes pod CIDR. cni := c.cfg.Installation.CNI if cni != nil && cni.IPAM != nil && cni.IPAM.Type == operatorv1.IPAMPluginHostLocal { diff --git a/pkg/render/windows.go b/pkg/render/windows.go index 19d8e1efb6..22ed57de2f 100644 --- a/pkg/render/windows.go +++ b/pkg/render/windows.go @@ -24,24 +24,35 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/components" "github.com/tigera/operator/pkg/controller/k8sapi" + "github.com/tigera/operator/pkg/imageoverride" rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/securitycontext" - "github.com/tigera/operator/pkg/tls/certificatemanagement" ) const ( WindowsNodeObjectName = "calico-node-windows" WindowsNodeMetricsService = "calico-node-metrics-windows" + + WindowsInstallCNIContainerName = "install-cni" + WindowsNodeContainerName = "node" + WindowsFelixContainerName = "felix" + WindowsConfdContainerName = "confd" ) +// WindowsNodeContainerNames are the calico-node-windows containers that always +// render. Confd is gated on BGP, so callers add it separately. +var WindowsNodeContainerNames = []string{ + WindowsNodeContainerName, + WindowsFelixContainerName, +} + func Windows( cfg *WindowsConfiguration, ) Component { @@ -49,14 +60,17 @@ func Windows( } type WindowsConfiguration struct { - K8sServiceEp k8sapi.ServiceEndpoint - K8sDNSServers []string - Installation *operatorv1.InstallationSpec - ClusterDomain string - TLS *TyphaNodeTLS - PrometheusServerTLS certificatemanagement.KeyPairInterface - NodeReporterMetricsPort int - VXLANVNI int + K8sServiceEp k8sapi.ServiceEndpoint + K8sDNSServers []string + Installation *operatorv1.InstallationSpec + ClusterDomain string + TLS *TyphaNodeTLS + VXLANVNI int + + // ImageOverrides lets a variant swap the windows node and CNI images. The + // controller wires in the operator's image overrides; nil resolves to the + // core images. + ImageOverrides *imageoverride.Overrides } type windowsComponent struct { @@ -77,13 +91,10 @@ func (c *windowsComponent) ResolveImages(is *operatorv1.ImageSet) error { return imageName } - if c.cfg.Installation.Variant.IsEnterprise() { - c.cniImage = appendIfErr(components.GetReference(components.ComponentTigeraCNIWindows, reg, path, prefix, is)) - c.nodeImage = appendIfErr(components.GetReference(components.ComponentTigeraNodeWindows, reg, path, prefix, is)) - } else { - c.cniImage = appendIfErr(components.GetReference(components.ComponentCalicoCNIWindows, reg, path, prefix, is)) - c.nodeImage = appendIfErr(components.GetReference(components.ComponentCalicoNodeWindows, reg, path, prefix, is)) - } + cniImage := c.cfg.ImageOverrides.Resolve(ComponentNameWindowsCNIImg, components.ComponentCalicoCNIWindows, c.cfg.Installation) + nodeImage := c.cfg.ImageOverrides.Resolve(ComponentNameWindowsNodeImg, components.ComponentCalicoNodeWindows, c.cfg.Installation) + c.cniImage = appendIfErr(components.GetReference(cniImage, reg, path, prefix, is)) + c.nodeImage = appendIfErr(components.GetReference(nodeImage, reg, path, prefix, is)) if len(errMsgs) != 0 { return fmt.Errorf("%s", strings.Join(errMsgs, ",")) @@ -91,6 +102,10 @@ func (c *windowsComponent) ResolveImages(is *operatorv1.ImageSet) error { return nil } +func (c *windowsComponent) WindowsConfig() *WindowsConfiguration { + return c.cfg +} + func (c *windowsComponent) SupportedOSType() rmeta.OSType { return rmeta.OSTypeWindows } @@ -116,11 +131,6 @@ func (c *windowsComponent) Objects() ([]client.Object, []client.Object) { objs := []client.Object{} - if c.cfg.Installation.Variant.IsEnterprise() { - // Include Service for exposing node metrics. - objs = append(objs, c.nodeMetricsService()) - } - cniConfig := c.windowsCNIConfigMap() if cniConfig != nil { objs = append(objs, cniConfig) @@ -135,43 +145,6 @@ func (c *windowsComponent) Ready() bool { return true } -// nodeMetricsService creates a Service which exposes two endpoints on calico/node for -// reporting Prometheus metrics (for policy enforcement activity and BGP stats). -// This service is used internally by Calico Enterprise and is separate from general -// Prometheus metrics which are user-configurable. -func (c *windowsComponent) nodeMetricsService() *corev1.Service { - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{ - Name: WindowsNodeMetricsService, - Namespace: common.CalicoNamespace, - Labels: map[string]string{"k8s-app": WindowsNodeObjectName}, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{"k8s-app": WindowsNodeObjectName}, - // Important: "None" tells Kubernetes that we want a headless service with - // no kube-proxy load balancer. If we omit this then kube-proxy will render - // a huge set of iptables rules for this service since there's an instance - // on every node. - ClusterIP: "None", - Ports: []corev1.ServicePort{ - { - Name: "calico-metrics-port", - Port: int32(c.cfg.NodeReporterMetricsPort), - TargetPort: intstr.FromInt(c.cfg.NodeReporterMetricsPort), - Protocol: corev1.ProtocolTCP, - }, - { - Name: "calico-bgp-metrics-port", - Port: nodeBGPReporterPort, - TargetPort: intstr.FromInt(int(nodeBGPReporterPort)), - Protocol: corev1.ProtocolTCP, - }, - }, - }, - } -} - // windowsCNIConfigMap returns a config map containing the CNI network config to be installed on each node. // Returns nil if no configmap is needed. func (c *windowsComponent) windowsCNIConfigMap() *corev1.ConfigMap { @@ -380,8 +353,8 @@ func (c *windowsComponent) windowsVolumes() []corev1.Volume { {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, c.cfg.TLS.TrustedBundle.Volume(), c.cfg.TLS.NodeSecret.Volume(), - corev1.Volume{Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - corev1.Volume{Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, + {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, + {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, } // If needed for this configuration, then include the CNI volumes. @@ -392,20 +365,6 @@ func (c *windowsComponent) windowsVolumes() []corev1.Volume { volumes = append(volumes, corev1.Volume{Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: c.cfg.Installation.WindowsNodes.CNILogDir, Type: &dirOrCreate}}}) } - // Override with Tigera-specific config. - if c.cfg.Installation.Variant.IsEnterprise() { - // Add volume for calico logs. - calicoLogVol := corev1.Volume{ - Name: "var-log-calico", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}, - } - volumes = append(volumes, calicoLogVol) - } - - if c.cfg.PrometheusServerTLS != nil { - volumes = append(volumes, c.cfg.PrometheusServerTLS.Volume()) - } - return volumes } @@ -458,7 +417,7 @@ func (c *windowsComponent) cniContainer() corev1.Container { // which ships the same binary at /CalicoWindows/calico.exe. Keep each // command's path in sync with the image it runs in. return corev1.Container{ - Name: "install-cni", + Name: WindowsInstallCNIContainerName, Image: c.cniImage, Command: []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/opt/cni/bin/calico.exe", "component", "cni", "install"}, Env: cniEnv, @@ -470,7 +429,7 @@ func (c *windowsComponent) cniContainer() corev1.Container { // nodeContainer creates the windows node startup container. func (c *windowsComponent) nodeContainer() corev1.Container { return corev1.Container{ - Name: "node", + Name: WindowsNodeContainerName, Image: c.nodeImage, Args: []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/node-service.ps1"}, WorkingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/", @@ -483,11 +442,10 @@ func (c *windowsComponent) nodeContainer() corev1.Container { // felixContainer creates the windows felix container. func (c *windowsComponent) felixContainer() corev1.Container { - lp, rp := c.windowsLivenessReadinessProbes() return corev1.Container{ - Name: "felix", + Name: WindowsFelixContainerName, Image: c.nodeImage, Args: []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/felix-service.ps1"}, WorkingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/", @@ -504,7 +462,7 @@ func (c *windowsComponent) felixContainer() corev1.Container { // confdContainer creates the windows confd container (used only for the windows-bgp backend). func (c *windowsComponent) confdContainer() corev1.Container { return corev1.Container{ - Name: "confd", + Name: WindowsConfdContainerName, Image: c.nodeImage, Args: []string{"$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/confd/confd-service.ps1"}, WorkingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/CalicoWindows/", @@ -663,31 +621,6 @@ func (c *windowsComponent) windowsEnvVars() []corev1.EnvVar { windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_IPV6SUPPORT", Value: "false"}) } - if c.cfg.Installation.Variant.IsEnterprise() { - // Add in Calico Enterprise specific configuration. - extraNodeEnv := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: fmt.Sprintf("%d", c.cfg.NodeReporterMetricsPort)}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - if c.cfg.PrometheusServerTLS != nil { - extraNodeEnv = append(extraNodeEnv, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCERTFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountCertificateFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERKEYFILE", Value: c.cfg.PrometheusServerTLS.VolumeMountKeyFilePath()}, - corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERCAFILE", Value: c.cfg.TLS.TrustedBundle.MountPath()}, - ) - } - windowsEnv = append(windowsEnv, extraNodeEnv...) - } - if c.cfg.Installation.NodeMetricsPort != nil { // If a node metrics port was given, then enable felix prometheus metrics and set the port. // Note that this takes precedence over any FelixConfiguration resources in the cluster. @@ -698,20 +631,6 @@ func (c *windowsComponent) windowsEnvVars() []corev1.EnvVar { windowsEnv = append(windowsEnv, extraNodeEnv...) } - // Configure provider specific environment variables here. - switch c.cfg.Installation.KubernetesProvider { - case operatorv1.ProviderOpenShift: - if c.cfg.Installation.Variant.IsEnterprise() { - // We need to configure a non-default trusted DNS server, since there's no kube-dns. - windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}) - } - case operatorv1.ProviderRKE2: - // For RKE2, configure a non-default trusted DNS server, as the DNS service is not named "kube-dns". - if c.cfg.Installation.Variant.IsEnterprise() { - windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}) - } - } - if c.cfg.Installation.CNI.Type != operatorv1.PluginCalico { windowsEnv = append(windowsEnv, corev1.EnvVar{Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}) } @@ -730,12 +649,7 @@ func (c *windowsComponent) windowsVolumeMounts() []corev1.VolumeMount { corev1.VolumeMount{MountPath: "/var/run/calico", Name: "var-run-calico"}, corev1.VolumeMount{MountPath: "/var/lib/calico", Name: "var-lib-calico"}) - if c.cfg.Installation.Variant.IsEnterprise() { - extraNodeMounts := []corev1.VolumeMount{ - {MountPath: "/var/log/calico", Name: "var-log-calico"}, - } - windowsVolumeMounts = append(windowsVolumeMounts, extraNodeMounts...) - } else if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { + if c.cfg.Installation.CNI.Type == operatorv1.PluginCalico { windowsVolumeMounts = append(windowsVolumeMounts, corev1.VolumeMount{MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}) } @@ -743,9 +657,6 @@ func (c *windowsComponent) windowsVolumeMounts() []corev1.VolumeMount { windowsVolumeMounts = append(windowsVolumeMounts, corev1.VolumeMount{MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}) } - if c.cfg.PrometheusServerTLS != nil { - windowsVolumeMounts = append(windowsVolumeMounts, c.cfg.PrometheusServerTLS.VolumeMount(c.SupportedOSType())) - } return windowsVolumeMounts } @@ -801,9 +712,6 @@ func (c *windowsComponent) windowsDaemonset(cniCfgMap *corev1.ConfigMap) *appsv1 initContainers := []corev1.Container{c.uninstallContainer()} annotations := c.cfg.TLS.TrustedBundle.HashAnnotations() - if c.cfg.PrometheusServerTLS != nil { - annotations[c.cfg.PrometheusServerTLS.HashAnnotationKey()] = c.cfg.PrometheusServerTLS.HashAnnotationValue() - } if cniCfgMap != nil { annotations[nodeCniConfigAnnotation] = rmeta.AnnotationHash(cniCfgMap.Data) diff --git a/pkg/render/windows_test.go b/pkg/render/windows_test.go index 73f333c94d..4f3224d16f 100644 --- a/pkg/render/windows_test.go +++ b/pkg/render/windows_test.go @@ -35,6 +35,7 @@ import ( "github.com/tigera/operator/pkg/controller/certificatemanager" "github.com/tigera/operator/pkg/controller/k8sapi" ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/imageoverride" "github.com/tigera/operator/pkg/render" rmeta "github.com/tigera/operator/pkg/render/common/meta" rtest "github.com/tigera/operator/pkg/render/common/test" @@ -107,12 +108,13 @@ var _ = Describe("Windows rendering tests", func() { // Create a default configuration. cfg = render.WindowsConfiguration{ - K8sServiceEp: k8sServiceEp, - K8sDNSServers: []string{"10.96.0.10"}, - Installation: defaultInstance, - ClusterDomain: defaultClusterDomain, - TLS: typhaNodeTLS, - VXLANVNI: 4096, + K8sServiceEp: k8sServiceEp, + K8sDNSServers: []string{"10.96.0.10"}, + Installation: defaultInstance, + ClusterDomain: defaultClusterDomain, + TLS: typhaNodeTLS, + VXLANVNI: 4096, + ImageOverrides: imageoverride.New(), } }) @@ -662,393 +664,6 @@ var _ = Describe("Windows rendering tests", func() { } }) - It("should render all resources for a default configuration using CalicoEnterprise", func() { - type testConf struct { - EnableBGP bool - EnableVXLAN bool - } - for _, testConfig := range []testConf{ - {true, false}, - {false, true}, - {true, true}, - } { - enableBGP := testConfig.EnableBGP - enableVXLAN := testConfig.EnableVXLAN - - if enableBGP { - defaultInstance.CalicoNetwork.BGP = &bgpEnabled - } else { - defaultInstance.CalicoNetwork.BGP = &bgpDisabled - } - - if enableVXLAN { - defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationVXLAN - } else { - defaultInstance.CalicoNetwork.IPPools[0].Encapsulation = operatorv1.EncapsulationNone - } - By(fmt.Sprintf("BGP enabled: %v, VXLAN enabled: %v", enableBGP, enableVXLAN), func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - defaultInstance.Variant = operatorv1.CalicoEnterprise - cfg.NodeReporterMetricsPort = 9081 - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - // The calico-node-windows daemonset has 3 containers (felix, node and confd). - // confd is only instantiated if using BGP. - numContainers := 3 - if !enableBGP { - numContainers = 2 - } - Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(numContainers)) - for _, container := range ds.Spec.Template.Spec.Containers { - - // Windows node image override results in correct image. - Expect(container.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(container.SecurityContext.Capabilities).To(BeNil()) - Expect(container.SecurityContext.Privileged).To(BeNil()) - Expect(container.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(container.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(container.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*container.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*container.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(container.SecurityContext.RunAsUser).To(BeNil()) - Expect(container.SecurityContext.RunAsGroup).To(BeNil()) - Expect(container.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(container.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(container.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(container.SecurityContext.ProcMount).To(BeNil()) - Expect(container.SecurityContext.SeccompProfile).To(BeNil()) - } - - felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - - // Windows node image override results in correct image. - Expect(felixContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(felixContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(felixContainer.SecurityContext.Privileged).To(BeNil()) - Expect(felixContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(felixContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(felixContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*felixContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*felixContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(felixContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(felixContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(felixContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(felixContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(felixContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(felixContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(felixContainer.SecurityContext.SeccompProfile).To(BeNil()) - - nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - - // Windows node image override results in correct image. - Expect(nodeContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(nodeContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(nodeContainer.SecurityContext.Privileged).To(BeNil()) - Expect(nodeContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(nodeContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(nodeContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*nodeContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*nodeContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(nodeContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(nodeContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(nodeContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(nodeContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(nodeContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(nodeContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(nodeContainer.SecurityContext.SeccompProfile).To(BeNil()) - - if enableBGP { - confdContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd") - - // Windows node image override results in correct image. - Expect(confdContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - Expect(confdContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(confdContainer.SecurityContext.Privileged).To(BeNil()) - Expect(confdContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(confdContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(confdContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*confdContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*confdContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(confdContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(confdContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(confdContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(confdContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(confdContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(confdContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(confdContainer.SecurityContext.SeccompProfile).To(BeNil()) - } - - // Validate correct number of init containers. - Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(2)) - - // CNI container uses image override. - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") - Expect(cniContainer.Image).To(Equal(components.TigeraRegistry + "tigera/cni-windows:" + components.ComponentTigeraCNIWindows.Version)) - - Expect(cniContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(cniContainer.SecurityContext.Privileged).To(BeNil()) - Expect(cniContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(cniContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(cniContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*cniContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*cniContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(cniContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(cniContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(cniContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(cniContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(cniContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(cniContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(cniContainer.SecurityContext.SeccompProfile).To(BeNil()) - - // uninstall container uses image override. - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(components.TigeraRegistry + "tigera/node-windows:" + components.ComponentTigeraNodeWindows.Version)) - - Expect(uninstallContainer.SecurityContext.Capabilities).To(BeNil()) - Expect(uninstallContainer.SecurityContext.Privileged).To(BeNil()) - Expect(uninstallContainer.SecurityContext.SELinuxOptions).To(BeNil()) - Expect(uninstallContainer.SecurityContext.WindowsOptions).To(Not(BeNil())) - Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpecName).To(BeNil()) - Expect(uninstallContainer.SecurityContext.WindowsOptions.GMSACredentialSpec).To(BeNil()) - Expect(*uninstallContainer.SecurityContext.WindowsOptions.RunAsUserName).To(Equal("NT AUTHORITY\\system")) - Expect(*uninstallContainer.SecurityContext.WindowsOptions.HostProcess).To(BeTrue()) - Expect(uninstallContainer.SecurityContext.RunAsUser).To(BeNil()) - Expect(uninstallContainer.SecurityContext.RunAsGroup).To(BeNil()) - Expect(uninstallContainer.SecurityContext.RunAsNonRoot).To(BeNil()) - Expect(uninstallContainer.SecurityContext.ReadOnlyRootFilesystem).To(BeNil()) - Expect(uninstallContainer.SecurityContext.AllowPrivilegeEscalation).To(BeNil()) - Expect(uninstallContainer.SecurityContext.ProcMount).To(BeNil()) - Expect(uninstallContainer.SecurityContext.SeccompProfile).To(BeNil()) - - // Verify env - expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - - {Name: "VXLAN_VNI", Value: "4096"}, - {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - } - - // Set CALICO_NETWORKING_BACKEND - if enableBGP { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}) - } else if enableVXLAN { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}) - } else { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CALICO_NETWORKING_BACKEND", Value: "none"}) - } - - // Set CLUSTER_TYPE - if enableBGP { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}) - } else { - expectedNodeEnv = append(expectedNodeEnv, corev1.EnvVar{Name: "CLUSTER_TYPE", Value: "k8s,operator,windows"}) - } - - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - if enableBGP { - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) - } - - // Expect the SECURITY_GROUP env variables to not be set - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - if enableBGP { - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - } - - expectedCNIEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, - {Name: "VXLAN_VNI", Value: "4096"}, - { - Name: "KUBERNETES_NODE_NAME", - Value: "", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "CNI_NETWORK_CONFIG", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - Key: "config", - LocalObjectReference: corev1.LocalObjectReference{ - Name: "cni-config-windows", - }, - }, - }, - }, - - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, - {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) - - expectedUninstallEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) - - // Verify volumes. - fileOrCreate := corev1.HostPathFileOrCreate - dirOrCreate := corev1.HostPathDirectoryOrCreate - expectedVols := []corev1.Volume{ - {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, - {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, - {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, - {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, - { - Name: "tigera-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ca-bundle", - }, - }, - }, - }, - { - Name: render.NodeTLSSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: render.NodeTLSSecretName, - DefaultMode: &defaultMode, - }, - }, - }, - {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, - } - Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) - - // Verify volume mounts. - expectedNodeVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - {MountPath: "/var/run/calico", Name: "var-run-calico"}, - {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, - {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, - {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico", Name: "var-log-calico", ReadOnly: false}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - if enableBGP { - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - } - - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) - - // Verify tolerations. - Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - - // Verify readiness and liveness probes. - verifyWindowsProbesAndLifecycle(ds, false) - }) - } - }) - It("should render all resources when using Calico CNI on EKS", func() { expectedResources := []struct { name string @@ -1162,240 +777,34 @@ var _ = Describe("Windows rendering tests", func() { // The calico-node-windows daemonset has 2 containers (felix, node) when using VXLAN Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(2)) - for _, container := range ds.Spec.Template.Spec.Containers { - // Windows image override results in correct image. - Expect(container.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - } - - // Validate correct number of init containers. - Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(2)) - - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") - - // CNI container uses image override. - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) - - // uninstall container uses image override. - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - - // Verify env - expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,ecs,windows"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, - }, - }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - - {Name: "VXLAN_VNI", Value: "4096"}, - {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - } - - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - - // Expect the SECURITY_GROUP env variables to not be set - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - - expectedCNIEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, - {Name: "VXLAN_VNI", Value: "4096"}, - - { - Name: "KUBERNETES_NODE_NAME", - Value: "", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, - }, - }, - { - Name: "CNI_NETWORK_CONFIG", - ValueFrom: &corev1.EnvVarSource{ - ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ - Key: "config", - LocalObjectReference: corev1.LocalObjectReference{ - Name: "cni-config-windows", - }, - }, - }, - }, - - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, - {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) - - expectedUninstallEnv := []corev1.EnvVar{ - {Name: "SLEEP", Value: "false"}, - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, - {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, - {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) - - // Verify volumes. - fileOrCreate := corev1.HostPathFileOrCreate - dirOrCreate := corev1.HostPathDirectoryOrCreate - expectedVols := []corev1.Volume{ - {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, - {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, - {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, - {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, - { - Name: "tigera-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ca-bundle", - }, - }, - }, - }, - { - Name: render.NodeTLSSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: render.NodeTLSSecretName, - DefaultMode: &defaultMode, - }, - }, - }, - } - Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) - - // Verify volume mounts. - expectedNodeVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - {MountPath: "/var/run/calico", Name: "var-run-calico"}, - {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, - {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, - {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, - {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) - - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) - - // Verify tolerations. - Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) - - // Verify readiness and liveness probes. - verifyWindowsProbesAndLifecycle(ds, true) - }) - - It("should properly render a configuration using the AmazonVPC CNI plugin", func() { - // Override the installation with one configured for AmazonVPC CNI. - amazonVPCInstalllation := &operatorv1.InstallationSpec{ - KubernetesProvider: operatorv1.ProviderEKS, - CNI: &operatorv1.CNISpec{Type: operatorv1.PluginAmazonVPC}, - ServiceCIDRs: []string{"10.96.0.0/12"}, - WindowsNodes: &operatorv1.WindowsNodeSpec{ - CNIBinDir: "/opt/cni/bin", - CNIConfigDir: "/etc/cni/net.d", - CNILogDir: "/var/log/calico/cni", - }, - } - cfg.Installation = amazonVPCInstalllation - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources - 1)) - - // Should render the correct resources. - Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - // Should not render CNI configuration. - cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") - Expect(cniCmResource).To(BeNil()) - - // The DaemonSet should have the correct configuration. - ds := dsResource.(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + for _, container := range ds.Spec.Template.Spec.Containers { + // Windows image override results in correct image. + Expect(container.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) + } + + // Validate correct number of init containers. + Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(2)) - // CNI install container should not be present. cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer).To(BeNil()) + rtest.ExpectEnv(cniContainer.Env, "CNI_NET_DIR", "/etc/cni/net.d") - // uninstall container should still be present. - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer).NotTo(BeNil()) + // CNI container uses image override. + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) - // Validate correct number of init containers. - Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) + // uninstall container uses image override. + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) // Verify env expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "AmazonVPC"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, + {Name: "CALICO_MANAGE_CNI", Value: "true"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "vxlan"}, {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, {Name: "CLUSTER_TYPE", Value: "k8s,operator,ecs,windows"}, - {Name: "IP", Value: "none"}, - {Name: "IP6", Value: "none"}, - {Name: "CALICO_MANAGE_CNI", Value: "false"}, {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, {Name: "FELIX_HEALTHENABLED", Value: "true"}, { Name: "NODENAME", @@ -1409,18 +818,20 @@ var _ = Describe("Windows rendering tests", func() { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }, }, + {Name: "IP", Value: "autodetect"}, + {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, + {Name: "IP6", Value: "none"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, {Name: "FELIX_TYPHACN", Value: "typha-server"}, {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}, - {Name: "FELIX_BPFEXTTOSERVICECONNMARK", Value: "0x80"}, {Name: "VXLAN_VNI", Value: "4096"}, {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "vpc.*"}, + {Name: "KUBE_NETWORK", Value: "Calico.*"}, {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, } @@ -1434,6 +845,49 @@ var _ = Describe("Windows rendering tests", func() { Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + expectedCNIEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/etc/cni/net.d"}, + {Name: "VXLAN_VNI", Value: "4096"}, + + { + Name: "KUBERNETES_NODE_NAME", + Value: "", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + }, + }, + { + Name: "CNI_NETWORK_CONFIG", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + Key: "config", + LocalObjectReference: corev1.LocalObjectReference{ + Name: "cni-config-windows", + }, + }, + }, + }, + + {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, + {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, + {Name: "KUBERNETES_SERVICE_CIDRS", Value: "10.96.0.0/12"}, + {Name: "KUBERNETES_DNS_SERVERS", Value: "10.96.0.10"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").Env).To(ConsistOf(expectedCNIEnv)) + + expectedUninstallEnv := []corev1.EnvVar{ + {Name: "SLEEP", Value: "false"}, + {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_BIN_DIR", Value: "/host/opt/cni/bin"}, + {Name: "CNI_CONF_NAME", Value: "10-calico.conflist"}, + {Name: "CNI_NET_DIR", Value: "/host/etc/cni/net.d"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").Env).To(ConsistOf(expectedUninstallEnv)) + // Verify volumes. fileOrCreate := corev1.HostPathFileOrCreate dirOrCreate := corev1.HostPathDirectoryOrCreate @@ -1442,6 +896,9 @@ var _ = Describe("Windows rendering tests", func() { {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, + {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, + {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, + {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, { Name: "tigera-ca-bundle", @@ -1467,14 +924,28 @@ var _ = Describe("Windows rendering tests", func() { // Verify volume mounts. expectedNodeVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, {MountPath: "/var/run/calico", Name: "var-run-calico"}, {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, + {MountPath: "/var/log/calico/cni", Name: "cni-log-dir", ReadOnly: false}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + expectedCNIVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) + + expectedUninstallVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, + {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + // Verify tolerations. Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) @@ -1482,163 +953,64 @@ var _ = Describe("Windows rendering tests", func() { verifyWindowsProbesAndLifecycle(ds, true) }) - DescribeTable("should properly render configuration using non-Calico CNI plugin", - func(cni operatorv1.CNIPluginType, ipam operatorv1.IPAMPluginType) { - installlation := &operatorv1.InstallationSpec{ - CNI: &operatorv1.CNISpec{ - Type: cni, - IPAM: &operatorv1.IPAMSpec{Type: ipam}, - }, - WindowsNodes: &operatorv1.WindowsNodeSpec{ - CNIBinDir: "/opt/cni/bin", - CNIConfigDir: "/etc/cni/net.d", - CNILogDir: "/var/log/calico/cni", - }, - } - cfg.Installation = installlation - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - - // Should render the correct resources. - Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - // Should not render CNI configuration. - cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") - Expect(cniCmResource).To(BeNil()) - - // The DaemonSet should have the correct configuration. - ds := dsResource.(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - // CNI install container should not be present. - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer).To(BeNil()) - // Validate correct number of init containers. - Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) - - // Verify env - expectedEnvs := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: string(cni)}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - } - for _, expected := range expectedEnvs { - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(expected)) - } - - // Verify readiness and liveness probes. - verifyWindowsProbesAndLifecycle(ds, true) - }, - Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal), - Entry("AmazonVPC", operatorv1.PluginAmazonVPC, operatorv1.IPAMPluginAmazonVPC), - Entry("AzureVNET", operatorv1.PluginAzureVNET, operatorv1.IPAMPluginAzureVNET), - ) - - It("should render all resources when running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, + It("should properly render a configuration using the AmazonVPC CNI plugin", func() { + // Override the installation with one configured for AmazonVPC CNI. + amazonVPCInstalllation := &operatorv1.InstallationSpec{ + KubernetesProvider: operatorv1.ProviderEKS, + CNI: &operatorv1.CNISpec{Type: operatorv1.PluginAmazonVPC}, + ServiceCIDRs: []string{"10.96.0.0/12"}, + WindowsNodes: &operatorv1.WindowsNodeSpec{ + CNIBinDir: "/opt/cni/bin", + CNIConfigDir: "/etc/cni/net.d", + CNILogDir: "/var/log/calico/cni", + }, } + cfg.Installation = amazonVPCInstalllation - defaultInstance.FlexVolumePath = "/etc/kubernetes/kubelet-plugins/volume/exec/" - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift component := render.Windows(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) + Expect(len(resources)).To(Equal(defaultNumExpectedResources - 1)) // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - Expect(felixContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - - // FIXME: confirm openshift CNI path defaults - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - - // FIXME: confirm openshift CNI path defaults - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) - // Verify volumes - // FIXME: confirm openshift CNI path defaults - fileOrCreate := corev1.HostPathFileOrCreate - dirOrCreate := corev1.HostPathDirectoryOrCreate - expectedVols := []corev1.Volume{ - {Name: "lib-modules", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/lib/modules"}}}, - {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, - {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, - {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, - {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, - { - Name: "tigera-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "tigera-ca-bundle", - }, - }, - }, - }, - { - Name: render.NodeTLSSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: render.NodeTLSSecretName, - DefaultMode: &defaultMode, - }, - }, - }, - } - Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) + // Should not render CNI configuration. + cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") + Expect(cniCmResource).To(BeNil()) + + // The DaemonSet should have the correct configuration. + ds := dsResource.(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + // CNI install container should not be present. + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer).To(BeNil()) + + // uninstall container should still be present. + uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") + Expect(uninstallContainer).NotTo(BeNil()) + + // Validate correct number of init containers. + Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) + // Verify env expectedNodeEnv := []corev1.EnvVar{ - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, + {Name: "CNI_PLUGIN_TYPE", Value: "AmazonVPC"}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,ecs,windows"}, + {Name: "IP", Value: "none"}, + {Name: "IP6", Value: "none"}, + {Name: "CALICO_MANAGE_CNI", Value: "false"}, {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + {Name: "FELIX_IPV6SUPPORT", Value: "false"}, {Name: "FELIX_HEALTHENABLED", Value: "true"}, { Name: "NODENAME", @@ -1652,92 +1024,32 @@ var _ = Describe("Windows rendering tests", func() { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }, }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, {Name: "FELIX_TYPHACN", Value: "typha-server"}, {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + {Name: "FELIX_ROUTESOURCE", Value: "WorkloadIPs"}, + {Name: "FELIX_BPFEXTTOSERVICECONNMARK", Value: "0x80"}, - // Calico Windows specific envvars {Name: "VXLAN_VNI", Value: "4096"}, {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, + {Name: "KUBE_NETWORK", Value: "vpc.*"}, {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) - - verifyWindowsProbesAndLifecycle(ds, true) - }) - - It("should render all resources when variant is CalicoEnterprise and running on openshift", func() { - expectedResources := []struct { - name string - ns string - group string - version string - kind string - }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, - {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, - {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, - } - - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift - cfg.NodeReporterMetricsPort = 9081 - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources))) - - // Should render the correct resources. - i := 0 - for _, expectedRes := range expectedResources { - rtest.ExpectResourceTypeAndObjectMetadata(resources[i], expectedRes.name, expectedRes.ns, expectedRes.group, expectedRes.version, expectedRes.kind) - i++ - } - - // The DaemonSet should have the correct configuration. - ds := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) - - // The pod template should have node critical priority - Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) - - felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) - nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) - cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) - uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) - - // FIXME: confirm openshift CNI path defaults - expectedCNIVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - // FIXME: confirm openshift CNI path defaults - expectedUninstallVolumeMounts := []corev1.VolumeMount{ - {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, - {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) + // Expect the SECURITY_GROUP env variables to not be set + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_DEFAULT_SECURITY_GROUPS")}))) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).NotTo(ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{"Name": Equal("TIGERA_POD_SECURITY_GROUP")}))) - // Verify volumes - // FIXME: confirm openshift CNI path defaults + // Verify volumes. fileOrCreate := corev1.HostPathFileOrCreate dirOrCreate := corev1.HostPathDirectoryOrCreate expectedVols := []corev1.Volume{ @@ -1745,9 +1057,6 @@ var _ = Describe("Windows rendering tests", func() { {Name: "var-run-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/calico", Type: &dirOrCreate}}}, {Name: "var-lib-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/lib/calico", Type: &dirOrCreate}}}, {Name: "xtables-lock", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/run/xtables.lock", Type: &fileOrCreate}}}, - {Name: "cni-bin-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/opt/cni/bin", Type: &dirOrCreate}}}, - {Name: "cni-net-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/cni/net.d"}}}, - {Name: "cni-log-dir", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico/cni", Type: &dirOrCreate}}}, {Name: "policysync", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/run/nodeagent", Type: &dirOrCreate}}}, { Name: "tigera-ca-bundle", @@ -1768,73 +1077,85 @@ var _ = Describe("Windows rendering tests", func() { }, }, }, - {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) - expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. - {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, - {Name: "DATASTORE_TYPE", Value: "kubernetes"}, - {Name: "WAIT_FOR_DATASTORE", Value: "true"}, - {Name: "CALICO_MANAGE_CNI", Value: "true"}, - {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, - {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, - {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, - {Name: "FELIX_HEALTHENABLED", Value: "true"}, - { - Name: "NODENAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "spec.nodeName"}, + // Verify volume mounts. + expectedNodeVolumeMounts := []corev1.VolumeMount{ + {MountPath: "/var/run/calico", Name: "var-run-calico"}, + {MountPath: "/var/lib/calico", Name: "var-lib-calico"}, + {MountPath: "c:/etc/pki/tls/certs", Name: "tigera-ca-bundle", ReadOnly: true}, + {MountPath: "c:/node-certs", Name: render.NodeTLSSecretName, ReadOnly: true}, + } + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").VolumeMounts).To(ConsistOf(expectedNodeVolumeMounts)) + + // Verify tolerations. + Expect(ds.Spec.Template.Spec.Tolerations).To(ConsistOf(rmeta.TolerateAll)) + + // Verify readiness and liveness probes. + verifyWindowsProbesAndLifecycle(ds, true) + }) + + DescribeTable("should properly render configuration using non-Calico CNI plugin", + func(cni operatorv1.CNIPluginType, ipam operatorv1.IPAMPluginType) { + installlation := &operatorv1.InstallationSpec{ + CNI: &operatorv1.CNISpec{ + Type: cni, + IPAM: &operatorv1.IPAMSpec{Type: ipam}, }, - }, - { - Name: "NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + WindowsNodes: &operatorv1.WindowsNodeSpec{ + CNIBinDir: "/opt/cni/bin", + CNIConfigDir: "/etc/cni/net.d", + CNILogDir: "/var/log/calico/cni", }, - }, - {Name: "IP", Value: "autodetect"}, - {Name: "IP_AUTODETECTION_METHOD", Value: "first-found"}, - {Name: "IP6", Value: "none"}, - {Name: "FELIX_IPV6SUPPORT", Value: "false"}, - {Name: "FELIX_TYPHAK8SNAMESPACE", Value: "calico-system"}, - {Name: "FELIX_TYPHAK8SSERVICENAME", Value: "calico-typha"}, - {Name: "FELIX_TYPHACAFILE", Value: certificatemanagement.TrustedCertBundleMountPath}, - {Name: "FELIX_TYPHACERTFILE", Value: "/node-certs/tls.crt"}, - {Name: "FELIX_TYPHACN", Value: "typha-server"}, - {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, + } + cfg.Installation = installlation - {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:openshift-dns/dns-default"}, + component := render.Windows(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, + // Should render the correct resources. + Expect(rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet")).ToNot(BeNil()) + dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") + Expect(dsResource).ToNot(BeNil()) - // Calico Windows specific envvars - {Name: "VXLAN_VNI", Value: "4096"}, - {Name: "VXLAN_ADAPTER", Value: ""}, - {Name: "KUBE_NETWORK", Value: "Calico.*"}, - {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, - {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, - } - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) - Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) + // Should not render CNI configuration. + cniCmResource := rtest.GetResource(resources, "cni-config-windows", "calico-system", "", "v1", "ConfigMap") + Expect(cniCmResource).To(BeNil()) - verifyWindowsProbesAndLifecycle(ds, false) - }) + // The DaemonSet should have the correct configuration. + ds := dsResource.(*appsv1.DaemonSet) + + // The pod template should have node critical priority + Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) + + // CNI install container should not be present. + cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") + Expect(cniContainer).To(BeNil()) + // Validate correct number of init containers. + Expect(len(ds.Spec.Template.Spec.InitContainers)).To(Equal(1)) + + // Verify env + expectedEnvs := []corev1.EnvVar{ + {Name: "CNI_PLUGIN_TYPE", Value: string(cni)}, + {Name: "CALICO_NETWORKING_BACKEND", Value: "none"}, + {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, + } + for _, expected := range expectedEnvs { + Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(expected)) + } + + // Verify readiness and liveness probes. + verifyWindowsProbesAndLifecycle(ds, true) + }, + Entry("GKE", operatorv1.PluginGKE, operatorv1.IPAMPluginHostLocal), + Entry("AmazonVPC", operatorv1.PluginAmazonVPC, operatorv1.IPAMPluginAmazonVPC), + Entry("AzureVNET", operatorv1.PluginAzureVNET, operatorv1.IPAMPluginAzureVNET), + ) - It("should render all resources when variant is CalicoEnterprise and running on RKE2", func() { + It("should render all resources when running on openshift", func() { expectedResources := []struct { name string ns string @@ -1842,19 +1163,16 @@ var _ = Describe("Windows rendering tests", func() { version string kind string }{ - {name: "calico-node-metrics-windows", ns: "calico-system", group: "", version: "v1", kind: "Service"}, {name: "cni-config-windows", ns: common.CalicoNamespace, group: "", version: "v1", kind: "ConfigMap"}, {name: common.WindowsDaemonSetName, ns: common.CalicoNamespace, group: "apps", version: "v1", kind: "DaemonSet"}, } - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.KubernetesProvider = operatorv1.ProviderRKE2 - cfg.NodeReporterMetricsPort = 9081 - + defaultInstance.FlexVolumePath = "/etc/kubernetes/kubelet-plugins/volume/exec/" + defaultInstance.KubernetesProvider = operatorv1.ProviderOpenShift component := render.Windows(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) resources, _ := component.Objects() - Expect(len(resources)).To(Equal(len(expectedResources)), fmt.Sprintf("Actual resources: %#v", resources)) + Expect(len(resources)).To(Equal(len(expectedResources))) // Should render the correct resources. i := 0 @@ -1870,22 +1188,22 @@ var _ = Describe("Windows rendering tests", func() { Expect(ds.Spec.Template.Spec.PriorityClassName).To(Equal(render.NodePriorityClassName)) felixContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix") - Expect(felixContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + Expect(felixContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) nodeContainer := rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node") - Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + Expect(nodeContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) cniContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni") - Expect(cniContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraCNIWindows.Image, components.ComponentTigeraCNIWindows.Version))) + Expect(cniContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoCNIWindows.Image, components.ComponentCalicoCNIWindows.Version))) uninstallContainer := rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico") - Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("%s%s%s:%s", components.TigeraRegistry, components.TigeraImagePath, components.ComponentTigeraNodeWindows.Image, components.ComponentTigeraNodeWindows.Version))) + Expect(uninstallContainer.Image).To(Equal(fmt.Sprintf("quay.io/%s%s:%s", components.CalicoImagePath, components.ComponentCalicoNodeWindows.Image, components.ComponentCalicoNodeWindows.Version))) - // FIXME: confirm RKE2 CNI path defaults + // FIXME: confirm openshift CNI path defaults expectedCNIVolumeMounts := []corev1.VolumeMount{ {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, } Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "install-cni").VolumeMounts).To(ConsistOf(expectedCNIVolumeMounts)) - // FIXME: confirm RKE2 CNI path defaults + // FIXME: confirm openshift CNI path defaults expectedUninstallVolumeMounts := []corev1.VolumeMount{ {MountPath: "/host/opt/cni/bin", Name: "cni-bin-dir"}, {MountPath: "/host/etc/cni/net.d", Name: "cni-net-dir"}, @@ -1893,7 +1211,7 @@ var _ = Describe("Windows rendering tests", func() { Expect(rtest.GetContainer(ds.Spec.Template.Spec.InitContainers, "uninstall-calico").VolumeMounts).To(ConsistOf(expectedUninstallVolumeMounts)) // Verify volumes - // FIXME: confirm RKE2 CNI path defaults + // FIXME: confirm openshift CNI path defaults fileOrCreate := corev1.HostPathFileOrCreate dirOrCreate := corev1.HostPathDirectoryOrCreate expectedVols := []corev1.Volume{ @@ -1924,18 +1242,16 @@ var _ = Describe("Windows rendering tests", func() { }, }, }, - {Name: "var-log-calico", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/var/log/calico", Type: &dirOrCreate}}}, } Expect(ds.Spec.Template.Spec.Volumes).To(ConsistOf(expectedVols)) expectedNodeEnv := []corev1.EnvVar{ - // Default envvars. {Name: "CNI_PLUGIN_TYPE", Value: "Calico"}, {Name: "DATASTORE_TYPE", Value: "kubernetes"}, {Name: "WAIT_FOR_DATASTORE", Value: "true"}, {Name: "CALICO_MANAGE_CNI", Value: "true"}, {Name: "CALICO_NETWORKING_BACKEND", Value: "windows-bgp"}, - {Name: "CLUSTER_TYPE", Value: "k8s,operator,bgp,windows"}, + {Name: "CLUSTER_TYPE", Value: "k8s,operator,openshift,bgp,windows"}, {Name: "CALICO_DISABLE_FILE_LOGGING", Value: "false"}, {Name: "FELIX_DEFAULTENDPOINTTOHOSTACTION", Value: "ACCEPT"}, {Name: "FELIX_HEALTHENABLED", Value: "true"}, @@ -1962,20 +1278,6 @@ var _ = Describe("Windows rendering tests", func() { {Name: "FELIX_TYPHACN", Value: "typha-server"}, {Name: "FELIX_TYPHAKEYFILE", Value: "/node-certs/tls.key"}, - {Name: "FELIX_DNSTRUSTEDSERVERS", Value: "k8s-service:kube-system/rke2-coredns-rke2-coredns"}, - - // Tigera-specific envvars - {Name: "FELIX_PROMETHEUSREPORTERENABLED", Value: "true"}, - {Name: "FELIX_PROMETHEUSREPORTERPORT", Value: "9081"}, - {Name: "FELIX_FLOWLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDELABELS", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDEPOLICIES", Value: "true"}, - {Name: "FELIX_FLOWLOGSFILEINCLUDESERVICE", Value: "true"}, - {Name: "FELIX_FLOWLOGSENABLENETWORKSETS", Value: "true"}, - {Name: "FELIX_FLOWLOGSCOLLECTPROCESSINFO", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEENABLED", Value: "true"}, - {Name: "FELIX_DNSLOGSFILEPERNODELIMIT", Value: "1000"}, - // Calico Windows specific envvars {Name: "VXLAN_VNI", Value: "4096"}, {Name: "VXLAN_ADAPTER", Value: ""}, @@ -1983,14 +1285,12 @@ var _ = Describe("Windows rendering tests", func() { {Name: "KUBERNETES_SERVICE_HOST", Value: "1.2.3.4"}, {Name: "KUBERNETES_SERVICE_PORT", Value: "6443"}, } - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(expectedNodeEnv)) - Expect(len(ds.Spec.Template.Spec.Containers[0].Env)).To(Equal(len(expectedNodeEnv))) - verifyWindowsProbesAndLifecycle(ds, false) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "node").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "felix").Env).To(ConsistOf(expectedNodeEnv)) + Expect(rtest.GetContainer(ds.Spec.Template.Spec.Containers, "confd").Env).To(ConsistOf(expectedNodeEnv)) - // The metrics service should have the correct configuration. - ms := rtest.GetResource(resources, "calico-node-metrics-windows", "calico-system", "", "v1", "Service").(*corev1.Service) - Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless to prevent kube-proxy from rendering too many iptables rules") + verifyWindowsProbesAndLifecycle(ds, true) }) Describe("AKS", func() { @@ -2133,55 +1433,6 @@ var _ = Describe("Windows rendering tests", func() { }) }) - It("should not enable prometheus metrics if NodeMetricsPort is nil", func() { - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.NodeMetricsPort = nil - cfg.NodeReporterMetricsPort = 9081 - - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) - - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - notExpectedEnvVar := corev1.EnvVar{Name: "FELIX_PROMETHEUSMETRICSPORT"} - ds := dsResource.(*appsv1.DaemonSet) - Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(notExpectedEnvVar)) - - // It should have the reporter port, though. - expected := corev1.EnvVar{Name: "FELIX_PROMETHEUSREPORTERPORT"} - Expect(ds.Spec.Template.Spec.Containers[0].Env).ToNot(ContainElement(expected)) - }) - - It("should set FELIX_PROMETHEUSMETRICSPORT with a custom value if NodeMetricsPort is set", func() { - var nodeMetricsPort int32 = 1234 - defaultInstance.Variant = operatorv1.CalicoEnterprise - defaultInstance.NodeMetricsPort = &nodeMetricsPort - component := render.Windows(&cfg) - Expect(component.ResolveImages(nil)).To(BeNil()) - resources, _ := component.Objects() - Expect(len(resources)).To(Equal(defaultNumExpectedResources + 1)) - - dsResource := rtest.GetResource(resources, "calico-node-windows", "calico-system", "apps", "v1", "DaemonSet") - Expect(dsResource).ToNot(BeNil()) - - // Assert on expected env vars. - expectedEnvVars := []corev1.EnvVar{ - {Name: "FELIX_PROMETHEUSMETRICSPORT", Value: "1234"}, - {Name: "FELIX_PROMETHEUSMETRICSENABLED", Value: "true"}, - } - ds := dsResource.(*appsv1.DaemonSet) - for _, v := range expectedEnvVars { - Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ContainElement(v)) - } - - // Assert we set annotations properly. - Expect(ds.Spec.Template.Annotations["prometheus.io/scrape"]).To(Equal("true")) - Expect(ds.Spec.Template.Annotations["prometheus.io/port"]).To(Equal("1234")) - }) - It("should render MaxUnavailable if a custom value was set", func() { two := intstr.FromInt(2) defaultInstance.NodeUpdateStrategy.RollingUpdate.MaxUnavailable = &two