diff --git a/cmd/cluster/gcp/create.go b/cmd/cluster/gcp/create.go index d16f3ca64ddd..08574b833108 100644 --- a/cmd/cluster/gcp/create.go +++ b/cmd/cluster/gcp/create.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "os" + "regexp" + "strings" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/cmd/cluster/core" @@ -20,6 +22,11 @@ import ( var _ core.Platform = (*CreateOptions)(nil) +var ( + gcpResourceLabelKeyPattern = regexp.MustCompile(`^[a-z]([_a-z0-9-]{0,61}[a-z0-9])?$`) + gcpResourceLabelValuePattern = regexp.MustCompile(`^$|^[a-z]([_a-z0-9-]{0,61}[a-z0-9])?$`) +) + const ( SATokenIssuerSecret = "sa-token-issuer-key" @@ -43,6 +50,7 @@ const ( flagZone = "zone" flagSubnet = "subnet" flagBootImage = "boot-image" + flagResourceLabels = "resource-labels" ) // RawCreateOptions contains the raw command-line options for creating a GCP cluster @@ -106,6 +114,9 @@ type RawCreateOptions struct { // BootImage is the GCP boot image for node instances. Overrides the default RHCOS image from the release payload BootImage string + + // ResourceLabels are labels applied to GCP resources created for the hosted cluster. + ResourceLabels []string } // BindOptions binds the GCP-specific flags to the provided flag set @@ -130,6 +141,7 @@ func BindOptions(opts *RawCreateOptions, flags *pflag.FlagSet) { flags.StringVar(&opts.Zone, flagZone, "", "GCP zone for node instances (e.g. us-central1-a). Required when --node-pool-replicas >= 0") flags.StringVar(&opts.Subnet, flagSubnet, "", "Subnet name for node instances. Defaults to the PSC subnet value") flags.StringVar(&opts.BootImage, flagBootImage, "", "GCP boot image for node instances. Overrides the default RHCOS image from the release payload") + flags.StringSliceVar(&opts.ResourceLabels, flagResourceLabels, opts.ResourceLabels, "Comma-separated GCP resource labels to apply to cluster resources (key=value). Can be specified multiple times.") } // ValidatedCreateOptions represents validated options for creating a GCP cluster @@ -277,6 +289,11 @@ func serviceAccountTokenIssuerSecret(namespace, name string) *corev1.Secret { // ApplyPlatformSpecifics applies GCP-specific configurations to the HostedCluster func (o *CreateOptions) ApplyPlatformSpecifics(hostedCluster *hyperv1.HostedCluster) error { + resourceLabels, err := parseResourceLabels(o.ResourceLabels) + if err != nil { + return err + } + hostedCluster.Spec.Platform.Type = hyperv1.GCPPlatform hostedCluster.Spec.Platform.GCP = &hyperv1.GCPPlatformSpec{ Project: o.Project, @@ -303,6 +320,7 @@ func (o *CreateOptions) ApplyPlatformSpecifics(hostedCluster *hyperv1.HostedClus }, }, EndpointAccess: hyperv1.GCPEndpointAccessType(o.EndpointAccess), + ResourceLabels: resourceLabels, } hostedCluster.Spec.IssuerURL = o.IssuerURL @@ -335,6 +353,30 @@ func (o *CreateOptions) ApplyPlatformSpecifics(hostedCluster *hyperv1.HostedClus return nil } +func parseResourceLabels(labels []string) ([]hyperv1.GCPResourceLabel, error) { + result := make([]hyperv1.GCPResourceLabel, 0, len(labels)) + seen := make(map[string]struct{}, len(labels)) + for _, label := range labels { + key, value, found := strings.Cut(label, "=") + if !found || key == "" { + return nil, fmt.Errorf("invalid --%s value %q: expected key=value", flagResourceLabels, label) + } + // GCP reserves the goog prefix for its own use; goog-partner-solution is the documented exception. + if !gcpResourceLabelKeyPattern.MatchString(key) || (strings.HasPrefix(key, "goog") && key != "goog-partner-solution") { + return nil, fmt.Errorf("invalid --%s value %q: invalid label key", flagResourceLabels, label) + } + if !gcpResourceLabelValuePattern.MatchString(value) { + return nil, fmt.Errorf("invalid --%s value %q: invalid label value", flagResourceLabels, label) + } + if _, exists := seen[key]; exists { + return nil, fmt.Errorf("invalid --%s value %q: duplicate key %q", flagResourceLabels, label, key) + } + seen[key] = struct{}{} + result = append(result, hyperv1.GCPResourceLabel{Key: key, Value: &value}) + } + return result, nil +} + // GenerateNodePools generates the NodePool resources for GCP func (o *CreateOptions) GenerateNodePools(constructor core.DefaultNodePoolConstructor) []*hyperv1.NodePool { nodePool := constructor(hyperv1.GCPPlatform, "") diff --git a/cmd/cluster/gcp/create_test.go b/cmd/cluster/gcp/create_test.go index 2470ee523118..f622ee9b9261 100644 --- a/cmd/cluster/gcp/create_test.go +++ b/cmd/cluster/gcp/create_test.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strings" "testing" . "github.com/onsi/gomega" @@ -41,6 +42,7 @@ func TestCreateOptionsApplyPlatformSpecifics(t *testing.T) { StorageServiceAccount: "storage@test-project-123.iam.gserviceaccount.com", ImageRegistryServiceAccount: "imageregistry@test-project-123.iam.gserviceaccount.com", NetworkServiceAccount: "network@test-project-123.iam.gserviceaccount.com", + ResourceLabels: []string{"hypershift-e2e=resource-labels"}, }, }, }, @@ -66,6 +68,77 @@ func TestCreateOptionsApplyPlatformSpecifics(t *testing.T) { g.Expect(hostedCluster.Spec.Platform.GCP.WorkloadIdentity.ServiceAccountsEmails.Storage).To(Equal(hyperv1.GCPServiceAccountEmail("storage@test-project-123.iam.gserviceaccount.com"))) g.Expect(hostedCluster.Spec.Platform.GCP.WorkloadIdentity.ServiceAccountsEmails.ImageRegistry).To(Equal(hyperv1.GCPServiceAccountEmail("imageregistry@test-project-123.iam.gserviceaccount.com"))) g.Expect(hostedCluster.Spec.Platform.GCP.WorkloadIdentity.ServiceAccountsEmails.Network).To(Equal(hyperv1.GCPServiceAccountEmail("network@test-project-123.iam.gserviceaccount.com"))) + g.Expect(hostedCluster.Spec.Platform.GCP.ResourceLabels).To(Equal([]hyperv1.GCPResourceLabel{{ + Key: "hypershift-e2e", Value: func() *string { value := "resource-labels"; return &value }(), + }})) +} + +func TestParseResourceLabels(t *testing.T) { + tests := map[string]struct { + labels []string + expected []hyperv1.GCPResourceLabel + expectError string + }{ + "When resource labels are valid, it should convert them to GCP resource labels": { + labels: []string{"environment=ci", "cost-center=cost1234", "empty="}, + expected: []hyperv1.GCPResourceLabel{ + {Key: "environment", Value: func() *string { value := "ci"; return &value }()}, + {Key: "cost-center", Value: func() *string { value := "cost1234"; return &value }()}, + {Key: "empty", Value: func() *string { value := ""; return &value }()}, + }, + }, + "When a resource label has no value separator, it should return an error": { + labels: []string{"environment"}, + expectError: "expected key=value", + }, + "When a resource label has an empty key, it should return an error": { + labels: []string{"=ci"}, + expectError: "expected key=value", + }, + "When resource label keys are duplicated, it should return an error": { + labels: []string{"environment=ci", "environment=test"}, + expectError: "duplicate key", + }, + "When a resource label key violates GCP requirements, it should return an error": { + labels: []string{"UPPER=value"}, + expectError: "invalid label key", + }, + "When a resource label key uses a reserved goog prefix, it should return an error": { + labels: []string{"goog-managed=value"}, + expectError: "invalid label key", + }, + "When a resource label value violates GCP requirements, it should return an error": { + labels: []string{"environment=value=with-equals"}, + expectError: "invalid label value", + }, + "When a resource label key exceeds GCP limits, it should return an error": { + labels: []string{strings.Repeat("a", 64) + "=value"}, + expectError: "invalid label key", + }, + "When a resource label value exceeds GCP limits, it should return an error": { + labels: []string{"environment=" + strings.Repeat("a", 64)}, + expectError: "invalid label value", + }, + "When a resource label uses the Google partner exception, it should be accepted": { + labels: []string{"goog-partner-solution=openshift"}, + expected: []hyperv1.GCPResourceLabel{ + {Key: "goog-partner-solution", Value: func() *string { value := "openshift"; return &value }()}, + }, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + g := NewGomegaWithT(t) + actual, err := parseResourceLabels(tc.labels) + if tc.expectError != "" { + g.Expect(err).To(MatchError(ContainSubstring(tc.expectError))) + return + } + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(actual).To(Equal(tc.expected)) + }) + } } func TestValidateGCPOptions(t *testing.T) { diff --git a/test/e2e/v2/internal/env_vars.go b/test/e2e/v2/internal/env_vars.go index 2b23a7948f7d..32c2ed323a28 100644 --- a/test/e2e/v2/internal/env_vars.go +++ b/test/e2e/v2/internal/env_vars.go @@ -153,6 +153,11 @@ func init() { "Management-cluster kubeconfig for the target role in a cross-cluster restore test.", false, ) + RegisterEnvVar( + "SHARED_DIR", + "CI shared directory containing platform-specific test configuration such as GCP workload identity credentials.", + false, + ) RegisterEnvVar( "E2E_SHOW_ENV_HELP", "When set to any non-empty value, displays environment variable help and exits without running tests.", diff --git a/test/e2e/v2/tests/hosted_cluster_gcp_test.go b/test/e2e/v2/tests/hosted_cluster_gcp_test.go new file mode 100644 index 000000000000..510669d1bdde --- /dev/null +++ b/test/e2e/v2/tests/hosted_cluster_gcp_test.go @@ -0,0 +1,228 @@ +//go:build e2ev2 + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "fmt" + "os" + "path" + "path/filepath" + "sort" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/gcputil" + "github.com/openshift/hypershift/test/e2e/v2/internal" + + corev1 "k8s.io/api/core/v1" + + "google.golang.org/api/compute/v1" + "google.golang.org/api/option" +) + +// RegisterHostedClusterGCPTests registers tests for GCP-specific hosted cluster resources. +func RegisterHostedClusterGCPTests(getTestCtx internal.TestContextGetter) { + GCPComputeResourceLabelsTest(getTestCtx) + GCPLoadBalancerServiceResourceLabelsTest(getTestCtx) +} + +// newGCPComputeClient creates a Compute client using the workload identity +// credentials made available to GCP E2E jobs. +func newGCPComputeClient(g Gomega, tc *internal.TestContext) *compute.Service { + GinkgoHelper() + sharedDir := internal.GetEnvVarValue("SHARED_DIR") + if sharedDir == "" { + Skip("SHARED_DIR is required to access GCP workload identity credentials") + } + credentialsFile := filepath.Join(sharedDir, gcpWIFCredentialsFile) + if _, err := os.Stat(credentialsFile); err != nil { + if os.IsNotExist(err) { + Skip(fmt.Sprintf("GCP workload identity credentials are unavailable at %s", credentialsFile)) + } + g.Expect(err).NotTo(HaveOccurred(), "failed to stat GCP workload identity credentials at %s", credentialsFile) + } + + computeService, err := compute.NewService(tc.Context, + option.WithAuthCredentialsFile(option.ExternalAccount, credentialsFile), + option.WithScopes(compute.ComputeScope), + ) + g.Expect(err).NotTo(HaveOccurred(), "failed to create GCP Compute client") + return computeService +} + +// GCPComputeResourceLabelsTest validates labels on NodePool VMs and their persistent disks. +func GCPComputeResourceLabelsTest(getTestCtx internal.TestContextGetter) { + Context("GCP compute resource labels", Label("GCP", "resource-labels"), func() { + BeforeEach(func() { + getTestCtx().SkipIfNotPlatform(hyperv1.GCPPlatform) + }) + + It("should apply HostedCluster resource labels to VMs and their persistent disks", Label("resource-labels", internal.InformingLabel), func() { + tc := getTestCtx() + hc, err := tc.GetHostedCluster() + Expect(err).NotTo(HaveOccurred(), "failed to get HostedCluster %s/%s", tc.ClusterNamespace, tc.ClusterName) + Expect(hc.Spec.Platform.GCP).NotTo(BeNil(), "HostedCluster %s/%s should have a GCP platform spec", hc.Namespace, hc.Name) + if len(hc.Spec.Platform.GCP.ResourceLabels) == 0 { + Skip("HostedCluster has no GCP resource labels configured") + } + + expectedLabels := make(map[string]string, len(hc.Spec.Platform.GCP.ResourceLabels)) + for _, label := range hc.Spec.Platform.GCP.ResourceLabels { + Expect(label.Value).NotTo(BeNil(), "GCP resource label %q should have a value", label.Key) + expectedLabels[label.Key] = *label.Value + } + + computeService := newGCPComputeClient(Default, tc) + + hostedClusterClient, err := tc.GetHostedClusterClient(hc) + Expect(err).NotTo(HaveOccurred(), "failed to create a client for HostedCluster %s/%s", hc.Namespace, hc.Name) + nodes := &corev1.NodeList{} + Expect(hostedClusterClient.List(tc.Context, nodes)).To(Succeed(), "failed to list nodes in HostedCluster %s/%s", hc.Namespace, hc.Name) + Expect(nodes.Items).NotTo(BeEmpty(), "expected at least one hosted cluster node") + + customerProjectID := hc.Spec.Platform.GCP.Project + Expect(customerProjectID).NotTo(BeEmpty(), "HostedCluster GCP project should be set") + for _, node := range nodes.Items { + providerIDParts := strings.Split(node.Spec.ProviderID, "/") + Expect(providerIDParts).To(HaveLen(5), "node %s providerID should have format gce:////", node.Name) + Expect(providerIDParts[0]).To(Equal("gce:"), "node %s providerID should be a GCE provider ID", node.Name) + Expect(providerIDParts[1]).To(BeEmpty(), "node %s providerID should have format gce:////", node.Name) + Expect(providerIDParts[2]).To(Equal(customerProjectID), "node %s providerID should reference the HostedCluster GCP project", node.Name) + + zone, instanceName := providerIDParts[3], providerIDParts[4] + Expect(zone).NotTo(BeEmpty(), "node %s providerID should include an instance zone", node.Name) + Expect(instanceName).NotTo(BeEmpty(), "node %s providerID should include an instance name", node.Name) + + instance, err := computeService.Instances.Get(customerProjectID, zone, instanceName).Context(tc.Context).Do() + Expect(err).NotTo(HaveOccurred(), "failed to get VM %s for node %s", instanceName, node.Name) + assertGCPResourceLabels(Default, instanceName, instance.Labels, expectedLabels) + + persistentDiskCount := 0 + for _, attachedDisk := range instance.Disks { + if attachedDisk.Source == "" { + continue + } + persistentDiskCount++ + diskName := path.Base(attachedDisk.Source) + disk, err := computeService.Disks.Get(customerProjectID, zone, diskName).Context(tc.Context).Do() + Expect(err).NotTo(HaveOccurred(), "failed to get persistent disk %s for VM %s", diskName, instanceName) + assertGCPResourceLabels(Default, diskName, disk.Labels, expectedLabels) + } + Expect(persistentDiskCount).To(BeNumerically(">", 0), "VM %s should have at least one persistent disk", instanceName) + } + }) + }) +} + +// GCPLoadBalancerServiceResourceLabelsTest validates the annotations that the +// hosted-cluster config operator uses to propagate resource labels to legacy +// GCP load balancers. +func GCPLoadBalancerServiceResourceLabelsTest(getTestCtx internal.TestContextGetter) { + Context("GCP load balancer Service resource labels", Label("GCP", "resource-labels"), func() { + BeforeEach(func() { + getTestCtx().SkipIfNotPlatform(hyperv1.GCPPlatform) + }) + + It("should apply HostedCluster resource labels to legacy load balancer Services", Label("resource-labels", internal.InformingLabel), func() { + tc := getTestCtx() + hc, err := tc.GetHostedCluster() + Expect(err).NotTo(HaveOccurred(), "failed to get HostedCluster %s/%s", tc.ClusterNamespace, tc.ClusterName) + Expect(hc.Spec.Platform.GCP).NotTo(BeNil(), "HostedCluster %s/%s should have a GCP platform spec", hc.Namespace, hc.Name) + if len(hc.Spec.Platform.GCP.ResourceLabels) == 0 { + Skip("HostedCluster has no GCP resource labels configured") + } + + expectedLabels := make(map[string]string, len(hc.Spec.Platform.GCP.ResourceLabels)) + for _, label := range hc.Spec.Platform.GCP.ResourceLabels { + Expect(label.Value).NotTo(BeNil(), "GCP resource label %q should have a value", label.Key) + expectedLabels[label.Key] = *label.Value + } + + hostedClusterClient, err := tc.GetHostedClusterClient(hc) + Expect(err).NotTo(HaveOccurred(), "failed to create a client for HostedCluster %s/%s", hc.Namespace, hc.Name) + services := &corev1.ServiceList{} + Expect(hostedClusterClient.List(tc.Context, services)).To(Succeed(), "failed to list Services in HostedCluster %s/%s", hc.Namespace, hc.Name) + Expect(services.Items).NotTo(BeEmpty(), "expected Services in HostedCluster %s/%s", hc.Namespace, hc.Name) + + legacyServiceCount := 0 + for i := range services.Items { + service := &services.Items[i] + if !isLegacyGCPLoadBalancerService(service) { + continue + } + legacyServiceCount++ + + actualLabels, annotationPresent, err := gcputil.ParseLBResourceLabelsAnnotation(service.Annotations) + Expect(err).NotTo(HaveOccurred(), "Service %s/%s has an invalid %s annotation", service.Namespace, service.Name, gcputil.LBResourceLabelsAnnotation) + Expect(annotationPresent).To(BeTrue(), "legacy GCP load balancer Service %s/%s should have %s annotation", service.Namespace, service.Name, gcputil.LBResourceLabelsAnnotation) + for key, value := range expectedLabels { + Expect(actualLabels).To(HaveKeyWithValue(key, value), "legacy GCP load balancer Service %s/%s should include label %s=%s", service.Namespace, service.Name, key, value) + } + + managedLabelKeys := strings.Split(service.Annotations[gcputil.ManagedLBResourceLabelsAnnotation], ",") + expectedLabelKeys := make([]string, 0, len(expectedLabels)) + for key := range expectedLabels { + expectedLabelKeys = append(expectedLabelKeys, key) + } + sort.Strings(expectedLabelKeys) + Expect(managedLabelKeys).To(Equal(expectedLabelKeys), "legacy GCP load balancer Service %s/%s should record all HyperShift-managed label keys", service.Namespace, service.Name) + } + Expect(legacyServiceCount).To(BeNumerically(">", 0), "expected at least one legacy GCP load balancer Service") + }) + }) +} + +func isLegacyGCPLoadBalancerService(service *corev1.Service) bool { + const ( + legacyRegionalInternalLoadBalancerClass = "networking.gke.io/l4-regional-internal-legacy" + legacyRegionalExternalLoadBalancerClass = "networking.gke.io/l4-regional-external-legacy" + l4RBSAnnotation = "cloud.google.com/l4-rbs" + l4RBSEnabled = "enabled" + netLBFinalizerV2 = "gke.networking.io/l4-netlb-v2" + netLBFinalizerV3 = "gke.networking.io/l4-netlb-v3" + ) + + if service.Spec.Type != corev1.ServiceTypeLoadBalancer { + return false + } + if service.Spec.LoadBalancerClass != nil { + return *service.Spec.LoadBalancerClass == legacyRegionalInternalLoadBalancerClass || *service.Spec.LoadBalancerClass == legacyRegionalExternalLoadBalancerClass + } + if service.Annotations[l4RBSAnnotation] == l4RBSEnabled { + return false + } + for _, finalizer := range service.Finalizers { + if finalizer == netLBFinalizerV2 || finalizer == netLBFinalizerV3 { + return false + } + } + return true +} + +var _ = Describe("[sig-hypershift][Jira:Hypershift][Feature:GCPComputeResources] Hosted Cluster GCP", Label("hosted-cluster-gcp"), func() { + var testCtx *internal.TestContext + + BeforeEach(func() { + testCtx = internal.GetTestContext() + Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + }) + + RegisterHostedClusterGCPTests(func() *internal.TestContext { return testCtx }) +}) diff --git a/test/e2e/v2/tests/hosted_cluster_psc_test.go b/test/e2e/v2/tests/hosted_cluster_psc_test.go index 28e8f43413af..afd6d6f16ee2 100644 --- a/test/e2e/v2/tests/hosted_cluster_psc_test.go +++ b/test/e2e/v2/tests/hosted_cluster_psc_test.go @@ -17,14 +17,33 @@ limitations under the License. package tests import ( + "fmt" + "os" + "path" + "path/filepath" + "strings" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/test/e2e/v2/internal" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "google.golang.org/api/compute/v1" +) + +const ( + gcpWIFCredentialsFile = "wif-cred.json" + gcpControlPlaneProjectIDFile = "control-plane-project-id" + backendServiceAnnotation = "service.kubernetes.io/backend-service" + tcpForwardingRuleAnnotation = "service.kubernetes.io/tcp-forwarding-rule" + gcpResourceLabelsCondition = "GCPResourceLabelsApplied" ) // GCPPrivateServiceConnectTest registers tests that validate PSC resource correctness on GCP. @@ -78,6 +97,139 @@ func GCPPrivateServiceConnectTest(getTestCtx internal.TestContextGetter) { }) } +// GCPResourceLabelsTest registers tests that validate labels on GCP resources +// created for Private Service Connect and router load balancers. +func GCPResourceLabelsTest(getTestCtx internal.TestContextGetter) { + Context("GCP resource labels", Label("GCP", "resource-labels"), func() { + BeforeEach(func() { + getTestCtx().SkipIfNotPlatform(hyperv1.GCPPlatform) + }) + + It("should apply HostedCluster resource labels to PSC and router forwarding resources", Label("resource-labels", internal.InformingLabel), func() { + tc := getTestCtx() + hc, err := tc.GetHostedCluster() + Expect(err).NotTo(HaveOccurred(), "failed to get HostedCluster %s/%s", tc.ClusterNamespace, tc.ClusterName) + Expect(hc.Spec.Platform.GCP).NotTo(BeNil(), "HostedCluster %s/%s should have a GCP platform spec", hc.Namespace, hc.Name) + if len(hc.Spec.Platform.GCP.ResourceLabels) == 0 { + Skip("HostedCluster has no GCP resource labels configured") + } + + expectedLabels := make(map[string]string, len(hc.Spec.Platform.GCP.ResourceLabels)) + for _, label := range hc.Spec.Platform.GCP.ResourceLabels { + Expect(label.Value).NotTo(BeNil(), "GCP resource label %q should have a value", label.Key) + expectedLabels[label.Key] = *label.Value + } + + sharedDir := internal.GetEnvVarValue("SHARED_DIR") + if sharedDir == "" { + Skip("SHARED_DIR is required to access GCP workload identity credentials") + } + controlPlaneProjectIDFile := filepath.Join(sharedDir, gcpControlPlaneProjectIDFile) + if _, err := os.Stat(controlPlaneProjectIDFile); err != nil { + if os.IsNotExist(err) { + Skip(fmt.Sprintf("GCP control-plane project ID is unavailable at %s", controlPlaneProjectIDFile)) + } + Expect(err).NotTo(HaveOccurred(), "failed to stat GCP control-plane project ID at %s", controlPlaneProjectIDFile) + } + controlPlaneProjectID, err := readGCPProjectID(controlPlaneProjectIDFile) + Expect(err).NotTo(HaveOccurred(), "failed to read GCP control-plane project ID from %s", controlPlaneProjectIDFile) + + computeService := newGCPComputeClient(Default, tc) + hcp := &hyperv1.HostedControlPlane{} + Expect(tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{Namespace: tc.ControlPlaneNamespace, Name: hc.Name}, hcp)).To(Succeed(), + "failed to get HostedControlPlane %s/%s", tc.ControlPlaneNamespace, hc.Name) + labelsCondition := meta.FindStatusCondition(hcp.Status.Conditions, gcpResourceLabelsCondition) + Expect(labelsCondition).NotTo(BeNil(), "HostedControlPlane should report %s", gcpResourceLabelsCondition) + Expect(labelsCondition.Status).To(Equal(metav1.ConditionTrue), "HostedControlPlane %s/%s resource label reconciliation should succeed: %s", + hcp.Namespace, hcp.Name, labelsCondition.Message) + + pscList := &hyperv1.GCPPrivateServiceConnectList{} + Expect(tc.MgmtClient.List(tc.Context, pscList, crclient.InNamespace(tc.ControlPlaneNamespace))).To(Succeed(), + "failed to list GCPPrivateServiceConnect resources in namespace %s", tc.ControlPlaneNamespace) + Expect(pscList.Items).To(HaveLen(1), "expected exactly one GCPPrivateServiceConnect in namespace %s", tc.ControlPlaneNamespace) + serviceAttachmentName := pscList.Items[0].Status.ServiceAttachmentName + Expect(serviceAttachmentName).NotTo(BeEmpty(), "GCPPrivateServiceConnect %s should have ServiceAttachmentName set", pscList.Items[0].Name) + + customerProjectID := hc.Spec.Platform.GCP.Project + region := hc.Spec.Platform.GCP.Region + Expect(customerProjectID).NotTo(BeEmpty(), "HostedCluster GCP project should be set") + Expect(region).NotTo(BeEmpty(), "HostedCluster GCP region should be set") + + pscAddressName := serviceAttachmentName + "-ip" + pscAddress, err := computeService.Addresses.Get(customerProjectID, region, pscAddressName).Context(tc.Context).Do() + Expect(err).NotTo(HaveOccurred(), "failed to get PSC Address %s", pscAddressName) + assertGCPResourceLabels(Default, pscAddressName, pscAddress.Labels, expectedLabels) + + pscForwardingRuleName := serviceAttachmentName + "-endpoint" + pscForwardingRule, err := computeService.ForwardingRules.Get(customerProjectID, region, pscForwardingRuleName).Context(tc.Context).Do() + Expect(err).NotTo(HaveOccurred(), "failed to get PSC ForwardingRule %s", pscForwardingRuleName) + assertGCPResourceLabels(Default, pscForwardingRuleName, pscForwardingRule.Labels, expectedLabels) + + foundRouterService := false + for _, serviceName := range []string{"router", "private-router"} { + routerService := &corev1.Service{} + err := tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{Namespace: tc.ControlPlaneNamespace, Name: serviceName}, routerService) + if apierrors.IsNotFound(err) { + continue + } + Expect(err).NotTo(HaveOccurred(), "failed to get %s Service in namespace %s", serviceName, tc.ControlPlaneNamespace) + foundRouterService = true + + if forwardingRuleName := routerService.Annotations[tcpForwardingRuleAnnotation]; forwardingRuleName != "" { + forwardingRuleName = path.Base(forwardingRuleName) + forwardingRule, err := computeService.ForwardingRules.Get(controlPlaneProjectID, region, forwardingRuleName).Context(tc.Context).Do() + Expect(err).NotTo(HaveOccurred(), "failed to get management-project forwarding rule %s for %s Service", forwardingRuleName, serviceName) + assertGCPResourceLabels(Default, forwardingRule.Name, forwardingRule.Labels, expectedLabels) + continue + } + + backendServiceName := routerService.Annotations[backendServiceAnnotation] + Expect(backendServiceName).NotTo(BeEmpty(), "%s Service should have %s or %s annotation", serviceName, backendServiceAnnotation, tcpForwardingRuleAnnotation) + + var matchingForwardingRules []*compute.ForwardingRule + pageToken := "" + for { + managementForwardingRules, err := computeService.ForwardingRules.List(controlPlaneProjectID, region).PageToken(pageToken).Context(tc.Context).Do() + Expect(err).NotTo(HaveOccurred(), "failed to list management-project forwarding rules") + for _, forwardingRule := range managementForwardingRules.Items { + if path.Base(forwardingRule.BackendService) == backendServiceName { + matchingForwardingRules = append(matchingForwardingRules, forwardingRule) + } + } + if managementForwardingRules.NextPageToken == "" { + break + } + pageToken = managementForwardingRules.NextPageToken + } + Expect(matchingForwardingRules).NotTo(BeEmpty(), "expected management-project forwarding rules for %s backend service %s", serviceName, backendServiceName) + for _, forwardingRule := range matchingForwardingRules { + assertGCPResourceLabels(Default, forwardingRule.Name, forwardingRule.Labels, expectedLabels) + } + } + Expect(foundRouterService).To(BeTrue(), "expected at least one router Service in namespace %s", tc.ControlPlaneNamespace) + }) + }) +} + +func assertGCPResourceLabels(g Gomega, resourceName string, actual, expected map[string]string) { + GinkgoHelper() + for key, value := range expected { + g.Expect(actual).To(HaveKeyWithValue(key, value), "GCP resource %s should have label %s=%s", resourceName, key, value) + } +} + +func readGCPProjectID(filename string) (string, error) { + contents, err := os.ReadFile(filename) + if err != nil { + return "", fmt.Errorf("read GCP project ID file %s: %w", filename, err) + } + projectID := strings.TrimSpace(string(contents)) + if projectID == "" { + return "", fmt.Errorf("GCP project ID file %s is empty", filename) + } + return projectID, nil +} + // RegisterGCPPSCTests registers all GCP Private Service Connect tests. func RegisterGCPPSCTests(getTestCtx internal.TestContextGetter) { GCPPrivateServiceConnectTest(getTestCtx) @@ -93,3 +245,14 @@ var _ = Describe("[sig-hypershift][Jira:Hypershift][Feature:GCPPrivateServiceCon RegisterGCPPSCTests(func() *internal.TestContext { return testCtx }) }) + +var _ = Describe("[sig-hypershift][Jira:Hypershift][Feature:GCPResourceLabels] GCP Resource Labels", Label("gcp-resource-labels"), func() { + var testCtx *internal.TestContext + + BeforeEach(func() { + testCtx = internal.GetTestContext() + Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + }) + + GCPResourceLabelsTest(func() *internal.TestContext { return testCtx }) +})