From 2bdaf08b3ceaa0dea3534ba4cbdb94c3e21df370 Mon Sep 17 00:00:00 2001 From: "Grzegorz (Greg) Barabasz" Date: Thu, 15 Oct 2026 15:34:17 +0100 Subject: [PATCH] feat(gcp): add resource tag support Add immutable GCP Resource Manager tags with project and organization parent support. Propagate tags to guest Infrastructure for PD CSI and image registry. Configure Tag User IAM and gate tagged clusters on CPO image capability. Signed-off-by: Grzegorz (Greg) Barabasz Commit-Message-Assisted-by: GPT-5.6-Terra (via Codex) --- Containerfile.control-plane | 1 + Dockerfile.control-plane | 1 + Dockerfile.dev | 1 + api/hypershift/v1beta1/gcp.go | 86 + api/hypershift/v1beta1/hostedcluster_types.go | 2 + .../v1beta1/zz_generated.deepcopy.go | 20 + .../GCPPlatform.yaml | 112 + .../GCPPlatform.yaml | 112 + .../hypershift/v1beta1/gcpplatformspec.go | 14 + .../hypershift/v1beta1/gcpresourcetag.go | 56 + client/applyconfiguration/utils.go | 2 + cmd/infra/gcp/iam-bindings.json | 5 +- cmd/infra/gcp/iam_test.go | 18 + ...hpreview.hostedclusters.gcp.testsuite.yaml | 3433 ++++++++++++++++- ...usters-Hypershift-CustomNoUpgrade.crd.yaml | 112 + ...s-Hypershift-TechPreviewNoUpgrade.crd.yaml | 112 + ...planes-Hypershift-CustomNoUpgrade.crd.yaml | 112 + ...s-Hypershift-TechPreviewNoUpgrade.crd.yaml | 112 + docs/content/how-to/gcp-platform.md | 114 + .../how-to/gcp/configure-image-registry.md | 12 + docs/content/how-to/gcp/create-gcp-iam.md | 36 +- docs/content/reference/aggregated-docs.md | 271 +- docs/content/reference/api.md | 109 + .../{gcp_credentials.go => gcp_conditions.go} | 14 + ...entials_test.go => gcp_conditions_test.go} | 51 +- .../hostedcluster/hostedcluster_controller.go | 8 +- .../hostedcluster_controller_test.go | 164 +- support/conditions/conditions_test.go | 8 + support/globalconfig/infrastructure.go | 19 + support/globalconfig/infrastructure_test.go | 70 + .../hypershift/api/hypershift/v1beta1/gcp.go | 86 + .../hypershift/v1beta1/hostedcluster_types.go | 2 + .../v1beta1/zz_generated.deepcopy.go | 20 + 33 files changed, 5284 insertions(+), 11 deletions(-) create mode 100644 client/applyconfiguration/hypershift/v1beta1/gcpresourcetag.go rename hypershift-operator/controllers/hostedcluster/{gcp_credentials.go => gcp_conditions.go} (56%) rename hypershift-operator/controllers/hostedcluster/{gcp_credentials_test.go => gcp_conditions_test.go} (80%) diff --git a/Containerfile.control-plane b/Containerfile.control-plane index 04e8820426fa..96717965a0ea 100644 --- a/Containerfile.control-plane +++ b/Containerfile.control-plane @@ -28,4 +28,5 @@ LABEL io.openshift.hypershift.restricted-psa=true LABEL io.openshift.hypershift.control-plane-pki-operator-signs-csrs=true LABEL io.openshift.hypershift.hosted-cluster-config-operator-reports-node-count=true LABEL io.openshift.hypershift.control-plane-operator-supports-kas-custom-kubeconfig=true +LABEL io.openshift.hypershift.control-plane-operator-supports-gcp-resource-tags=true LABEL io.openshift.hypershift.control-plane-operator.v2-isdefault=true diff --git a/Dockerfile.control-plane b/Dockerfile.control-plane index fdea2e18c84a..f114edea7791 100644 --- a/Dockerfile.control-plane +++ b/Dockerfile.control-plane @@ -28,4 +28,5 @@ LABEL io.openshift.hypershift.restricted-psa=true LABEL io.openshift.hypershift.control-plane-pki-operator-signs-csrs=true LABEL io.openshift.hypershift.hosted-cluster-config-operator-reports-node-count=true LABEL io.openshift.hypershift.control-plane-operator-supports-kas-custom-kubeconfig=true +LABEL io.openshift.hypershift.control-plane-operator-supports-gcp-resource-tags=true LABEL io.openshift.hypershift.control-plane-operator.v2-isdefault=true diff --git a/Dockerfile.dev b/Dockerfile.dev index 3789aaaabe4d..3639dcfe792c 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -41,4 +41,5 @@ LABEL io.openshift.hypershift.restricted-psa=true LABEL io.openshift.hypershift.control-plane-pki-operator-signs-csrs=true LABEL io.openshift.hypershift.hosted-cluster-config-operator-reports-node-count=true LABEL io.openshift.hypershift.control-plane-operator-supports-kas-custom-kubeconfig=true +LABEL io.openshift.hypershift.control-plane-operator-supports-gcp-resource-tags=true LABEL io.openshift.hypershift.control-plane-operator.v2-isdefault=true diff --git a/api/hypershift/v1beta1/gcp.go b/api/hypershift/v1beta1/gcp.go index b298c7acdf91..f415f8c08656 100644 --- a/api/hypershift/v1beta1/gcp.go +++ b/api/hypershift/v1beta1/gcp.go @@ -47,6 +47,50 @@ type GCPResourceLabel struct { Value *string `json:"value,omitempty"` } +// GCPResourceTag selects a pre-existing Google Cloud Resource Manager tag by +// its parent, short key, and short value. The tag must be defined in the +// HostedCluster project or an organization before the cluster is created. +// See https://cloud.google.com/resource-manager/docs/tags/tags-overview. +type GCPResourceTag struct { + // parentID is the ID of the project or organization that defines the TagKey. + // When omitted, the HostedCluster's GCP project is used. + // An explicit value must be the HostedCluster's GCP project ID or a numeric + // organization ID. A numeric value is treated as an organization ID; project + // numbers are not supported. + // An organization ID consists of 1 to 32 decimal digits and must not start with 0. + // A project ID is 6 to 30 characters long, contains only lowercase letters, digits + // and hyphens, starts with a letter and must not end with a hyphen. + // + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:XValidation:rule="self.matches('(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)')",message="parentID must be a numeric organization ID or a valid GCP project ID" + ParentID string `json:"parentID,omitempty"` + + // key is required and is the short name of the pre-existing Resource Manager TagKey. + // A key is 1-63 characters, begins and ends with + // an ASCII alphanumeric character, and may contain letters, digits, '.', + // '_', or '-' between them. + // + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:XValidation:rule="self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$')",message="key must start and end with an ASCII letter or digit and contain only ASCII letters, digits, '.', '_' or '-'" + Key string `json:"key,omitempty"` + + // value is required and is the short name of the pre-existing Resource Manager TagValue for + // key. Exactly one value for a TagKey can be attached to a resource. A value + // is 1-63 characters, begins and ends with an ASCII alphanumeric character, + // and may contain ASCII letters and digits, `_-.@%=+:,*#&()[]{}`, and + // spaces between them. + // + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:XValidation:rule="self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\\\\[\\\\]{}\\\\- ]{0,61}[a-zA-Z0-9])?$')",message="value must start and end with an ASCII letter or digit and contain only ASCII letters, digits, spaces or the characters _-.@%=+:,*#&()[]{}" + Value string `json:"value,omitempty"` +} + // GCPEndpointAccessType defines the endpoint access type for GCP clusters. // Equivalent to AWS EndpointAccessType but adapted for GCP networking model. type GCPEndpointAccessType string @@ -107,6 +151,8 @@ type GCPNetworkConfig struct { // +kubebuilder:validation:XValidation:rule="self.workloadIdentity.serviceAccountsEmails.storage.contains('@') && self.workloadIdentity.serviceAccountsEmails.storage.endsWith('@' + self.project + '.iam.gserviceaccount.com')",message="storage service account must belong to the same project" // +kubebuilder:validation:XValidation:rule="self.workloadIdentity.serviceAccountsEmails.imageRegistry.contains('@') && self.workloadIdentity.serviceAccountsEmails.imageRegistry.endsWith('@' + self.project + '.iam.gserviceaccount.com')",message="imageRegistry service account must belong to the same project" // +kubebuilder:validation:XValidation:rule="self.workloadIdentity.serviceAccountsEmails.network.endsWith('@' + self.project + '.iam.gserviceaccount.com')",message="network service account must belong to the same project" +// +kubebuilder:validation:XValidation:rule="has(self.resourceTags) == has(oldSelf.resourceTags)",message="resourceTags may only be configured during installation",fieldPath=".resourceTags" +// +kubebuilder:validation:XValidation:rule="!has(self.resourceTags) || self.resourceTags.all(t, !has(t.parentID) || t.parentID.matches('^[1-9][0-9]{0,31}$') || t.parentID == self.project)",message="resourceTags parentID must be an organization ID or the GCP platform project",fieldPath=".resourceTags" type GCPPlatformSpec struct { // project is the GCP project ID. // A valid project ID must satisfy the following rules: @@ -161,6 +207,45 @@ type GCPPlatformSpec struct { // +kubebuilder:validation:MaxItems=60 ResourceLabels []GCPResourceLabel `json:"resourceLabels,omitempty"` + // resourceTags are pre-existing Google Cloud Resource Manager tags to apply to + // GCP resources created for the hosted cluster. + // Currently, tags are applied to newly provisioned persistent disks and, when the + // image registry is enabled, to its storage bucket. Other GCP resources, such as + // worker VMs and their boot disks, are not tagged. + // The set of tagged resources is subject to change over time. + // + // When omitted, no tags are applied through this setting. When specified, + // the list must contain 1 to 50 entries, and each short key must be unique + // across the list, even when the tags have different parents. A tag's + // parentID defaults to the HostedCluster's GCP project if omitted. + // Google Cloud permits at most 50 directly attached tags per resource, + // including tags attached by other actors. Coordinate with external tagging + // automation to leave sufficient capacity; admission does not reserve tag slots. + // + // The referenced tags must already exist, and the storage and imageRegistry + // service accounts must have permission to use them. Neither is checked when the + // HostedCluster is created. When a tag cannot be applied, provisioning of the + // affected disk or bucket can fail, and the error is reported in the persistent + // volume claim's events or the image registry operator's status. + // + // resourceTags can only be set when the HostedCluster is created. It cannot be + // added, removed or changed afterwards. If the configured list must change, + // the HostedCluster must be recreated. + // Resource tags require a control plane operator image that supports GCP resource + // tag propagation. When that capability is unavailable, reconciliation fails + // with a message identifying resourceTags on the ReconciliationSucceeded + // condition, and dependent component deployments are blocked until a + // supporting control plane operator image is configured. + // + // +optional + // +listType=atomic + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=50 + // +immutable + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="resourceTags are immutable" + // +kubebuilder:validation:XValidation:rule="self.all(t, self.exists_one(u, u.key == t.key))",message="resourceTags keys must be unique across all parents" + ResourceTags []GCPResourceTag `json:"resourceTags,omitempty"` + // workloadIdentity configures Workload Identity Federation for the cluster. // This enables secure, short-lived token-based authentication without storing // long-term service account keys. These fields are immutable after cluster creation @@ -349,6 +434,7 @@ type GCPServiceAccountsEmails struct { // that manages GCS storage for the internal container image registry. // This GSA requires the following IAM roles: // - roles/storage.admin (Storage Admin - for creating and managing GCS buckets and objects) + // - roles/resourcemanager.tagUser (Tag User - for applying resource tags to the bucket) // See cmd/infra/gcp/iam-bindings.json for the authoritative role definitions. // Format: service-account-name@project-id.iam.gserviceaccount.com // diff --git a/api/hypershift/v1beta1/hostedcluster_types.go b/api/hypershift/v1beta1/hostedcluster_types.go index ff15d4f1719d..97cffb545fd2 100644 --- a/api/hypershift/v1beta1/hostedcluster_types.go +++ b/api/hypershift/v1beta1/hostedcluster_types.go @@ -1486,6 +1486,8 @@ func PlatformTypes() []PlatformType { // PlatformSpec specifies the underlying infrastructure provider for the cluster // and is used to configure platform specific behavior. +// +// +openshift:validation:FeatureGateAwareXValidation:featureGate=GCPPlatform,rule="(has(self.gcp) && has(self.gcp.resourceTags)) == (has(oldSelf.gcp) && has(oldSelf.gcp.resourceTags))",message="resourceTags may only be configured during installation" type PlatformSpec struct { // type is the type of infrastructure provider for the cluster. // diff --git a/api/hypershift/v1beta1/zz_generated.deepcopy.go b/api/hypershift/v1beta1/zz_generated.deepcopy.go index 4c0459433bcf..6b2522e73d84 100644 --- a/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -2146,6 +2146,11 @@ func (in *GCPPlatformSpec) DeepCopyInto(out *GCPPlatformSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ResourceTags != nil { + in, out := &in.ResourceTags, &out.ResourceTags + *out = make([]GCPResourceTag, len(*in)) + copy(*out, *in) + } out.WorkloadIdentity = in.WorkloadIdentity } @@ -2302,6 +2307,21 @@ func (in *GCPResourceReference) DeepCopy() *GCPResourceReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GCPResourceTag) DeepCopyInto(out *GCPResourceTag) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCPResourceTag. +func (in *GCPResourceTag) DeepCopy() *GCPResourceTag { + if in == nil { + return nil + } + out := new(GCPResourceTag) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GCPServiceAccountsEmails) DeepCopyInto(out *GCPServiceAccountsEmails) { *out = *in diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml index cdcc4808aca4..619db6595a48 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml @@ -6105,6 +6105,104 @@ spec: x-kubernetes-list-map-keys: - key x-kubernetes-list-type: map + resourceTags: + description: |- + resourceTags are pre-existing Google Cloud Resource Manager tags to apply to + GCP resources created for the hosted cluster. + Currently, tags are applied to newly provisioned persistent disks and, when the + image registry is enabled, to its storage bucket. Other GCP resources, such as + worker VMs and their boot disks, are not tagged. + The set of tagged resources is subject to change over time. + + When omitted, no tags are applied through this setting. When specified, + the list must contain 1 to 50 entries, and each short key must be unique + across the list, even when the tags have different parents. A tag's + parentID defaults to the HostedCluster's GCP project if omitted. + Google Cloud permits at most 50 directly attached tags per resource, + including tags attached by other actors. Coordinate with external tagging + automation to leave sufficient capacity; admission does not reserve tag slots. + + The referenced tags must already exist, and the storage and imageRegistry + service accounts must have permission to use them. Neither is checked when the + HostedCluster is created. When a tag cannot be applied, provisioning of the + affected disk or bucket can fail, and the error is reported in the persistent + volume claim's events or the image registry operator's status. + + resourceTags can only be set when the HostedCluster is created. It cannot be + added, removed or changed afterwards. If the configured list must change, + the HostedCluster must be recreated. + Resource tags require a control plane operator image that supports GCP resource + tag propagation. When that capability is unavailable, reconciliation fails + with a message identifying resourceTags on the ReconciliationSucceeded + condition, and dependent component deployments are blocked until a + supporting control plane operator image is configured. + items: + description: |- + GCPResourceTag selects a pre-existing Google Cloud Resource Manager tag by + its parent, short key, and short value. The tag must be defined in the + HostedCluster project or an organization before the cluster is created. + See https://cloud.google.com/resource-manager/docs/tags/tags-overview. + properties: + key: + description: |- + key is required and is the short name of the pre-existing Resource Manager TagKey. + A key is 1-63 characters, begins and ends with + an ASCII alphanumeric character, and may contain letters, digits, '.', + '_', or '-' between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: key must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + '.', '_' or '-' + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$') + parentID: + description: |- + parentID is the ID of the project or organization that defines the TagKey. + When omitted, the HostedCluster's GCP project is used. + An explicit value must be the HostedCluster's GCP project ID or a numeric + organization ID. A numeric value is treated as an organization ID; project + numbers are not supported. + An organization ID consists of 1 to 32 decimal digits and must not start with 0. + A project ID is 6 to 30 characters long, contains only lowercase letters, digits + and hyphens, starts with a letter and must not end with a hyphen. + maxLength: 32 + minLength: 1 + type: string + x-kubernetes-validations: + - message: parentID must be a numeric organization ID + or a valid GCP project ID + rule: self.matches('(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)') + value: + description: |- + value is required and is the short name of the pre-existing Resource Manager TagValue for + key. Exactly one value for a TagKey can be attached to a resource. A value + is 1-63 characters, begins and ends with an ASCII alphanumeric character, + and may contain ASCII letters and digits, `_-.@%=+:,*#&()[]{}`, and + spaces between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: value must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + spaces or the characters _-.@%=+:,*#&()[]{} + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\\[\\]{}\\- + ]{0,61}[a-zA-Z0-9])?$') + required: + - key + - value + type: object + maxItems: 50 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: resourceTags are immutable + rule: self == oldSelf + - message: resourceTags keys must be unique across all parents + rule: self.all(t, self.exists_one(u, u.key == t.key)) workloadIdentity: description: |- workloadIdentity configures Workload Identity Federation for the cluster. @@ -6245,6 +6343,7 @@ spec: that manages GCS storage for the internal container image registry. This GSA requires the following IAM roles: - roles/storage.admin (Storage Admin - for creating and managing GCS buckets and objects) + - roles/resourcemanager.tagUser (Tag User - for applying resource tags to the bucket) See cmd/infra/gcp/iam-bindings.json for the authoritative role definitions. Format: service-account-name@project-id.iam.gserviceaccount.com @@ -6379,6 +6478,15 @@ spec: - message: network service account must belong to the same project rule: self.workloadIdentity.serviceAccountsEmails.network.endsWith('@' + self.project + '.iam.gserviceaccount.com') + - fieldPath: .resourceTags + message: resourceTags may only be configured during installation + rule: has(self.resourceTags) == has(oldSelf.resourceTags) + - fieldPath: .resourceTags + message: resourceTags parentID must be an organization ID or + the GCP platform project + rule: '!has(self.resourceTags) || self.resourceTags.all(t, !has(t.parentID) + || t.parentID.matches(''^[1-9][0-9]{0,31}$'') || t.parentID + == self.project)' ibmcloud: description: ibmcloud defines IBMCloud specific settings for components properties: @@ -6860,6 +6968,10 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: resourceTags may only be configured during installation + rule: (has(self.gcp) && has(self.gcp.resourceTags)) == (has(oldSelf.gcp) + && has(oldSelf.gcp.resourceTags)) pullSecret: description: |- pullSecret is a local reference to a Secret that must have a ".dockerconfigjson" key whose content must be a valid Openshift pull secret JSON. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml index 3e87f5df92b1..5f6f8cd97efb 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml @@ -5982,6 +5982,104 @@ spec: x-kubernetes-list-map-keys: - key x-kubernetes-list-type: map + resourceTags: + description: |- + resourceTags are pre-existing Google Cloud Resource Manager tags to apply to + GCP resources created for the hosted cluster. + Currently, tags are applied to newly provisioned persistent disks and, when the + image registry is enabled, to its storage bucket. Other GCP resources, such as + worker VMs and their boot disks, are not tagged. + The set of tagged resources is subject to change over time. + + When omitted, no tags are applied through this setting. When specified, + the list must contain 1 to 50 entries, and each short key must be unique + across the list, even when the tags have different parents. A tag's + parentID defaults to the HostedCluster's GCP project if omitted. + Google Cloud permits at most 50 directly attached tags per resource, + including tags attached by other actors. Coordinate with external tagging + automation to leave sufficient capacity; admission does not reserve tag slots. + + The referenced tags must already exist, and the storage and imageRegistry + service accounts must have permission to use them. Neither is checked when the + HostedCluster is created. When a tag cannot be applied, provisioning of the + affected disk or bucket can fail, and the error is reported in the persistent + volume claim's events or the image registry operator's status. + + resourceTags can only be set when the HostedCluster is created. It cannot be + added, removed or changed afterwards. If the configured list must change, + the HostedCluster must be recreated. + Resource tags require a control plane operator image that supports GCP resource + tag propagation. When that capability is unavailable, reconciliation fails + with a message identifying resourceTags on the ReconciliationSucceeded + condition, and dependent component deployments are blocked until a + supporting control plane operator image is configured. + items: + description: |- + GCPResourceTag selects a pre-existing Google Cloud Resource Manager tag by + its parent, short key, and short value. The tag must be defined in the + HostedCluster project or an organization before the cluster is created. + See https://cloud.google.com/resource-manager/docs/tags/tags-overview. + properties: + key: + description: |- + key is required and is the short name of the pre-existing Resource Manager TagKey. + A key is 1-63 characters, begins and ends with + an ASCII alphanumeric character, and may contain letters, digits, '.', + '_', or '-' between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: key must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + '.', '_' or '-' + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$') + parentID: + description: |- + parentID is the ID of the project or organization that defines the TagKey. + When omitted, the HostedCluster's GCP project is used. + An explicit value must be the HostedCluster's GCP project ID or a numeric + organization ID. A numeric value is treated as an organization ID; project + numbers are not supported. + An organization ID consists of 1 to 32 decimal digits and must not start with 0. + A project ID is 6 to 30 characters long, contains only lowercase letters, digits + and hyphens, starts with a letter and must not end with a hyphen. + maxLength: 32 + minLength: 1 + type: string + x-kubernetes-validations: + - message: parentID must be a numeric organization ID + or a valid GCP project ID + rule: self.matches('(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)') + value: + description: |- + value is required and is the short name of the pre-existing Resource Manager TagValue for + key. Exactly one value for a TagKey can be attached to a resource. A value + is 1-63 characters, begins and ends with an ASCII alphanumeric character, + and may contain ASCII letters and digits, `_-.@%=+:,*#&()[]{}`, and + spaces between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: value must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + spaces or the characters _-.@%=+:,*#&()[]{} + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\\[\\]{}\\- + ]{0,61}[a-zA-Z0-9])?$') + required: + - key + - value + type: object + maxItems: 50 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: resourceTags are immutable + rule: self == oldSelf + - message: resourceTags keys must be unique across all parents + rule: self.all(t, self.exists_one(u, u.key == t.key)) workloadIdentity: description: |- workloadIdentity configures Workload Identity Federation for the cluster. @@ -6122,6 +6220,7 @@ spec: that manages GCS storage for the internal container image registry. This GSA requires the following IAM roles: - roles/storage.admin (Storage Admin - for creating and managing GCS buckets and objects) + - roles/resourcemanager.tagUser (Tag User - for applying resource tags to the bucket) See cmd/infra/gcp/iam-bindings.json for the authoritative role definitions. Format: service-account-name@project-id.iam.gserviceaccount.com @@ -6256,6 +6355,15 @@ spec: - message: network service account must belong to the same project rule: self.workloadIdentity.serviceAccountsEmails.network.endsWith('@' + self.project + '.iam.gserviceaccount.com') + - fieldPath: .resourceTags + message: resourceTags may only be configured during installation + rule: has(self.resourceTags) == has(oldSelf.resourceTags) + - fieldPath: .resourceTags + message: resourceTags parentID must be an organization ID or + the GCP platform project + rule: '!has(self.resourceTags) || self.resourceTags.all(t, !has(t.parentID) + || t.parentID.matches(''^[1-9][0-9]{0,31}$'') || t.parentID + == self.project)' ibmcloud: description: ibmcloud defines IBMCloud specific settings for components properties: @@ -6737,6 +6845,10 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: resourceTags may only be configured during installation + rule: (has(self.gcp) && has(self.gcp.resourceTags)) == (has(oldSelf.gcp) + && has(oldSelf.gcp.resourceTags)) pullSecret: description: pullSecret is a reference to a secret containing the pull secret for the hosted control plane. diff --git a/client/applyconfiguration/hypershift/v1beta1/gcpplatformspec.go b/client/applyconfiguration/hypershift/v1beta1/gcpplatformspec.go index 254cc2660fa7..d657c0ffa917 100644 --- a/client/applyconfiguration/hypershift/v1beta1/gcpplatformspec.go +++ b/client/applyconfiguration/hypershift/v1beta1/gcpplatformspec.go @@ -29,6 +29,7 @@ type GCPPlatformSpecApplyConfiguration struct { NetworkConfig *GCPNetworkConfigApplyConfiguration `json:"networkConfig,omitempty"` EndpointAccess *hypershiftv1beta1.GCPEndpointAccessType `json:"endpointAccess,omitempty"` ResourceLabels []GCPResourceLabelApplyConfiguration `json:"resourceLabels,omitempty"` + ResourceTags []GCPResourceTagApplyConfiguration `json:"resourceTags,omitempty"` WorkloadIdentity *GCPWorkloadIdentityConfigApplyConfiguration `json:"workloadIdentity,omitempty"` } @@ -83,6 +84,19 @@ func (b *GCPPlatformSpecApplyConfiguration) WithResourceLabels(values ...*GCPRes return b } +// WithResourceTags adds the given value to the ResourceTags field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the ResourceTags field. +func (b *GCPPlatformSpecApplyConfiguration) WithResourceTags(values ...*GCPResourceTagApplyConfiguration) *GCPPlatformSpecApplyConfiguration { + for i := range values { + if values[i] == nil { + panic("nil value passed to WithResourceTags") + } + b.ResourceTags = append(b.ResourceTags, *values[i]) + } + return b +} + // WithWorkloadIdentity sets the WorkloadIdentity field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the WorkloadIdentity field is set to the value of the last call. diff --git a/client/applyconfiguration/hypershift/v1beta1/gcpresourcetag.go b/client/applyconfiguration/hypershift/v1beta1/gcpresourcetag.go new file mode 100644 index 000000000000..dc34e4783449 --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/gcpresourcetag.go @@ -0,0 +1,56 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// GCPResourceTagApplyConfiguration represents a declarative configuration of the GCPResourceTag type for use +// with apply. +type GCPResourceTagApplyConfiguration struct { + ParentID *string `json:"parentID,omitempty"` + Key *string `json:"key,omitempty"` + Value *string `json:"value,omitempty"` +} + +// GCPResourceTagApplyConfiguration constructs a declarative configuration of the GCPResourceTag type for use with +// apply. +func GCPResourceTag() *GCPResourceTagApplyConfiguration { + return &GCPResourceTagApplyConfiguration{} +} + +// WithParentID sets the ParentID field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ParentID field is set to the value of the last call. +func (b *GCPResourceTagApplyConfiguration) WithParentID(value string) *GCPResourceTagApplyConfiguration { + b.ParentID = &value + return b +} + +// WithKey sets the Key field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Key field is set to the value of the last call. +func (b *GCPResourceTagApplyConfiguration) WithKey(value string) *GCPResourceTagApplyConfiguration { + b.Key = &value + return b +} + +// WithValue sets the Value field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Value field is set to the value of the last call. +func (b *GCPResourceTagApplyConfiguration) WithValue(value string) *GCPResourceTagApplyConfiguration { + b.Value = &value + return b +} diff --git a/client/applyconfiguration/utils.go b/client/applyconfiguration/utils.go index a4262dc4176f..1e63e295aeaf 100644 --- a/client/applyconfiguration/utils.go +++ b/client/applyconfiguration/utils.go @@ -237,6 +237,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &hypershiftv1beta1.GCPResourceLabelApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("GCPResourceReference"): return &hypershiftv1beta1.GCPResourceReferenceApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("GCPResourceTag"): + return &hypershiftv1beta1.GCPResourceTagApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("GCPServiceAccountsEmails"): return &hypershiftv1beta1.GCPServiceAccountsEmailsApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("GCPWorkloadIdentityConfig"): diff --git a/cmd/infra/gcp/iam-bindings.json b/cmd/infra/gcp/iam-bindings.json index 7fc5df9bdc33..9986cfb4e485 100644 --- a/cmd/infra/gcp/iam-bindings.json +++ b/cmd/infra/gcp/iam-bindings.json @@ -58,7 +58,10 @@ "name": "image-registry", "displayName": "HyperShift Image Registry Operator", "description": "Service account for Image Registry Operator GCS storage operations", - "roles": ["roles/storage.admin"], + "roles": [ + "roles/storage.admin", + "roles/resourcemanager.tagUser" + ], "k8sServiceAccounts": [ {"namespace": "openshift-image-registry", "name": "cluster-image-registry-operator"}, {"namespace": "openshift-image-registry", "name": "registry"} diff --git a/cmd/infra/gcp/iam_test.go b/cmd/infra/gcp/iam_test.go index f3f8d737bb5e..13c38fcf0977 100644 --- a/cmd/infra/gcp/iam_test.go +++ b/cmd/infra/gcp/iam_test.go @@ -400,6 +400,24 @@ func TestLoadServiceAccountDefinitions(t *testing.T) { K8sServiceAccountRef{Namespace: "openshift-image-registry", Name: "registry"}, )) }) + + t.Run("When loading tag-consuming definitions, it should grant Tag User to storage and image-registry", func(t *testing.T) { + g := NewWithT(t) + definitions, err := loadServiceAccountDefinitions() + g.Expect(err).NotTo(HaveOccurred()) + + for _, name := range []string{"gcp-pd-csi", "image-registry"} { + var definition *ServiceAccountDefinition + for i := range definitions { + if definitions[i].Name == name { + definition = &definitions[i] + break + } + } + g.Expect(definition).NotTo(BeNil(), "expected to find %s service account definition", name) + g.Expect(definition.Roles).To(ContainElement("roles/resourcemanager.tagUser"), "%s needs Tag User for resource tags", name) + } + }) } func TestIsTransientIAMError(t *testing.T) { diff --git a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/techpreview.hostedclusters.gcp.testsuite.yaml b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/techpreview.hostedclusters.gcp.testsuite.yaml index 1aba436ec5d9..907eb9396b58 100644 --- a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/techpreview.hostedclusters.gcp.testsuite.yaml +++ b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/techpreview.hostedclusters.gcp.testsuite.yaml @@ -1137,7 +1137,3250 @@ tests: type: Route route: {} + # --- GCP resourceTags validation --- + - name: When GCP resourceTags contain a valid tag it should succeed + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: "Cost.Center_123" + value: "prod@eu-west1" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + - name: When a GCP resourceTag key is empty it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: "" + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + expectedError: "key in body should be at least 1 chars long" + + - name: When a GCP resourceTag value is empty it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: "" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + expectedError: "value in body should be at least 1 chars long" + + - name: When a GCP resourceTag key contains a forbidden character it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment/invalid + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "key must start and end with an ASCII letter or digit and contain only ASCII letters, digits, '.', '_' or '-'" + + - name: When a GCP resourceTag value contains an invalid character it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production/blue + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "value must start and end with an ASCII letter or digit and contain only ASCII letters, digits, spaces or the characters _-.@%=+:,*#&()[]{}" + + - name: When a GCP resourceTag value does not start with a letter or number it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: "-production" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "value must start and end with an ASCII letter or digit and contain only ASCII letters, digits, spaces or the characters _-.@%=+:,*#&()[]{}" + + - name: When a GCP resourceTag key contains a single quote it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: "environment'prod" + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "key must start and end with an ASCII letter or digit and contain only ASCII letters, digits, '.', '_' or '-'" + + - name: When a GCP resourceTag key contains a double quote it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: "environment\"prod" + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "key must start and end with an ASCII letter or digit and contain only ASCII letters, digits, '.', '_' or '-'" + + - name: When a GCP resourceTag key contains a backslash it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: "environment\\prod" + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "key must start and end with an ASCII letter or digit and contain only ASCII letters, digits, '.', '_' or '-'" + + - name: When a GCP resourceTag value contains a single quote it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: "production'blue" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "value must start and end with an ASCII letter or digit and contain only ASCII letters, digits, spaces or the characters _-.@%=+:,*#&()[]{}" + + - name: When a GCP resourceTag value contains a double quote it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: "production\"blue" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "value must start and end with an ASCII letter or digit and contain only ASCII letters, digits, spaces or the characters _-.@%=+:,*#&()[]{}" + + - name: When a GCP resourceTag value contains a backslash it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: "production\\blue" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "value must start and end with an ASCII letter or digit and contain only ASCII letters, digits, spaces or the characters _-.@%=+:,*#&()[]{}" + + - name: When GCP resourceTags contain duplicate keys it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + - key: environment + value: development + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags keys must be unique across all parents" + + - name: When GCP resourceTags contain 63-character keys and values it should succeed + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1 + value: Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1 + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + - name: When a GCP resourceTag key exceeds 63 characters it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "key: Too long" + + - name: When a GCP resourceTag value exceeds 63 characters it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "value: Too long" + + - name: When a GCP resourceTag key starts with a non-alphanumeric ASCII character it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: "-env" + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "key must start and end with an ASCII letter or digit and contain only ASCII letters, digits, '.', '_' or '-'" + + - name: When a GCP resourceTag key ends with a non-alphanumeric ASCII character it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: "env." + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "key must start and end with an ASCII letter or digit and contain only ASCII letters, digits, '.', '_' or '-'" + + - name: When a GCP resourceTag value ends with a non-alphanumeric ASCII character it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: "prod-" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "value must start and end with an ASCII letter or digit and contain only ASCII letters, digits, spaces or the characters _-.@%=+:,*#&()[]{}" + - name: When a GCP resourceTag key starts with a non-ASCII character it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: "équipe" + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "key must start and end with an ASCII letter or digit and contain only ASCII letters, digits, '.', '_' or '-'" + + - name: When a GCP resourceTag value starts with a non-ASCII character it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: "équipe" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "value must start and end with an ASCII letter or digit and contain only ASCII letters, digits, spaces or the characters _-.@%=+:,*#&()[]{}" + + - name: When GCP resourceTags contain 51 entries it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: key0 + value: production + - key: key1 + value: production + - key: key2 + value: production + - key: key3 + value: production + - key: key4 + value: production + - key: key5 + value: production + - key: key6 + value: production + - key: key7 + value: production + - key: key8 + value: production + - key: key9 + value: production + - key: key10 + value: production + - key: key11 + value: production + - key: key12 + value: production + - key: key13 + value: production + - key: key14 + value: production + - key: key15 + value: production + - key: key16 + value: production + - key: key17 + value: production + - key: key18 + value: production + - key: key19 + value: production + - key: key20 + value: production + - key: key21 + value: production + - key: key22 + value: production + - key: key23 + value: production + - key: key24 + value: production + - key: key25 + value: production + - key: key26 + value: production + - key: key27 + value: production + - key: key28 + value: production + - key: key29 + value: production + - key: key30 + value: production + - key: key31 + value: production + - key: key32 + value: production + - key: key33 + value: production + - key: key34 + value: production + - key: key35 + value: production + - key: key36 + value: production + - key: key37 + value: production + - key: key38 + value: production + - key: key39 + value: production + - key: key40 + value: production + - key: key41 + value: production + - key: key42 + value: production + - key: key43 + value: production + - key: key44 + value: production + - key: key45 + value: production + - key: key46 + value: production + - key: key47 + value: production + - key: key48 + value: production + - key: key49 + value: production + - key: key50 + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags: Too many" + + - name: When GCP resourceTags are empty it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: [] + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags in body should have at least 1 items" + + - name: When a GCP resourceTag value contains supported punctuation it should succeed + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: ENV_TEST.1 + value: "a_-.@%=+:,*#&()[]{} b" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + - name: When a GCP resourceTag value contains a tab it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: ENV_TEST.1 + value: "prod\tblue" + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "value must start and end with an ASCII letter or digit and contain only ASCII letters, digits, spaces or the characters _-.@%=+:,*#&()[]{}" + + - name: When GCP resourceTags set the own project as parentID it should succeed + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: my-project-123 + key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + - name: When GCP resourceTags set an organization parentID it should succeed + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: "123456789012" + key: cost-center + value: shared + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + - name: When GCP resourceTags have distinct keys across omitted, project, and organization parents it should succeed + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + - parentID: my-project-123 + key: team + value: platform + - parentID: "123456789012" + key: cost-center + value: shared + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + - name: When GCP resourceTags have distinct keys across organizations it should succeed + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: "123456789012" + key: team + value: platform + - parentID: "987654321098" + key: cost-center + value: shared + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + - name: When GCP resourceTags set a foreign project parentID it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: another-project + key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags parentID must be an organization ID or the GCP platform project" + + - name: When GCP resourceTags set a malformed parentID it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: Not-A-Project + key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "parentID must be a numeric organization ID or a valid GCP project ID" + + - name: When GCP resourceTags reference the Google system namespace it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: "google" + key: "system:sdp-data-sensitivity" + value: HIGH + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags parentID must be an organization ID or the GCP platform project" + + - name: When GCP resourceTags reference the Google App Hub namespace it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: "google:AppHub" + key: environment + value: PRODUCTION + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "parentID must be a numeric organization ID or a valid GCP project ID" + + - name: When GCP resourceTags set an organization parentID with a leading zero it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: "0123456789" + key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "parentID must be a numeric organization ID or a valid GCP project ID" + + - name: When GCP resourceTags set an empty parentID it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: "" + key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "parentID in body should be at least 1 chars long" + + - name: When GCP resourceTags repeat a key with omitted and explicit project parents it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + - parentID: my-project-123 + key: environment + value: development + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags keys must be unique across all parents" + + - name: When GCP resourceTags repeat a key across project and organization parents it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + - parentID: "123456789012" + key: environment + value: shared + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags keys must be unique across all parents" + + - name: When GCP resourceTags repeat a key across organizations it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: "123456789012" + key: environment + value: production + - parentID: "987654321098" + key: environment + value: shared + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags keys must be unique across all parents" + onUpdate: + # --- GCP resourceTags installation-time immutability --- + - name: When GCP resourceTags are changed it should reject the update + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: staging + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags are immutable" + + - name: When GCP resourceTag parentID changes it should reject the update + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - parentID: "123456789012" + key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags are immutable" + + - name: When GCP resourceTags are reordered it should reject the update + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + - key: team + value: platform + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: team + value: platform + - key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags are immutable" + + - name: When GCP resourceTags are added to a nonempty list it should reject the update + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + - key: team + value: platform + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags are immutable" + + - name: When GCP resourceTags are present and the entire gcp object is removed it should reject the update + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags may only be configured during installation" + + - name: When GCP resourceTags are added with the entire gcp object it should reject the update + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags may only be configured during installation" + + - name: When GCP resourceTags are removed it should reject the update + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags may only be configured during installation" + + - name: When GCP resourceTags are added after creation it should reject the update + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "resourceTags may only be configured during installation" + # --- GCP network service account immutability --- - name: When GCP network service account is changed it should fail initial: | @@ -1427,6 +4670,194 @@ tests: servicePublishingStrategy: route: {} type: Route + - name: When updating release image while GCP resourceTags are unchanged it should pass + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: GCP + gcp: + project: my-project-123 + region: us-central1 + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + workloadIdentity: + projectNumber: "123456789012" + poolID: my-wif-pool + providerID: my-wif-provider + serviceAccountsEmails: + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.16.0-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expected: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + capabilities: {} + controllerAvailabilityPolicy: HighlyAvailable + dns: + baseDomain: example.com + etcd: + managed: + storage: + persistentVolume: + size: 8Gi + type: PersistentVolume + managementType: Managed + infrastructureAvailabilityPolicy: SingleReplica + issuerURL: https://kubernetes.default.svc + networking: + clusterNetwork: + - cidr: 10.132.0.0/14 + networkType: OVNKubernetes + serviceNetwork: + - cidr: 172.31.0.0/16 + olmCatalogPlacement: management + platform: + gcp: + endpointAccess: Private + networkConfig: + network: + name: my-network + privateServiceConnectSubnet: + name: my-psc-subnet + project: my-project-123 + region: us-central1 + workloadIdentity: + poolID: my-wif-pool + projectNumber: '123456789012' + providerID: my-wif-provider + serviceAccountsEmails: + cloudController: cloudcontroller@my-project-123.iam.gserviceaccount.com + controlPlane: controlplane@my-project-123.iam.gserviceaccount.com + imageRegistry: imageregistry@my-project-123.iam.gserviceaccount.com + network: cnccsa@my-project-123.iam.gserviceaccount.com + nodePool: nodepool@my-project-123.iam.gserviceaccount.com + storage: storage@my-project-123.iam.gserviceaccount.com + resourceTags: + - key: environment + value: production + type: GCP + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.16.0-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + route: {} + type: Route + - service: OAuthServer + servicePublishingStrategy: + route: {} + type: Route + - service: Konnectivity + servicePublishingStrategy: + route: {} + type: Route + - service: Ignition + servicePublishingStrategy: + route: {} + type: Route # --- GCP network service account ratcheting --- - name: When updating a previously invalid network SA to a new invalid value it should fail initialCRDPatches: @@ -2134,4 +5565,4 @@ tests: - service: Ignition servicePublishingStrategy: route: {} - type: Route \ No newline at end of file + type: Route diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml index f22efa3043c1..8db55affe648 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml @@ -8570,6 +8570,104 @@ spec: x-kubernetes-list-map-keys: - key x-kubernetes-list-type: map + resourceTags: + description: |- + resourceTags are pre-existing Google Cloud Resource Manager tags to apply to + GCP resources created for the hosted cluster. + Currently, tags are applied to newly provisioned persistent disks and, when the + image registry is enabled, to its storage bucket. Other GCP resources, such as + worker VMs and their boot disks, are not tagged. + The set of tagged resources is subject to change over time. + + When omitted, no tags are applied through this setting. When specified, + the list must contain 1 to 50 entries, and each short key must be unique + across the list, even when the tags have different parents. A tag's + parentID defaults to the HostedCluster's GCP project if omitted. + Google Cloud permits at most 50 directly attached tags per resource, + including tags attached by other actors. Coordinate with external tagging + automation to leave sufficient capacity; admission does not reserve tag slots. + + The referenced tags must already exist, and the storage and imageRegistry + service accounts must have permission to use them. Neither is checked when the + HostedCluster is created. When a tag cannot be applied, provisioning of the + affected disk or bucket can fail, and the error is reported in the persistent + volume claim's events or the image registry operator's status. + + resourceTags can only be set when the HostedCluster is created. It cannot be + added, removed or changed afterwards. If the configured list must change, + the HostedCluster must be recreated. + Resource tags require a control plane operator image that supports GCP resource + tag propagation. When that capability is unavailable, reconciliation fails + with a message identifying resourceTags on the ReconciliationSucceeded + condition, and dependent component deployments are blocked until a + supporting control plane operator image is configured. + items: + description: |- + GCPResourceTag selects a pre-existing Google Cloud Resource Manager tag by + its parent, short key, and short value. The tag must be defined in the + HostedCluster project or an organization before the cluster is created. + See https://cloud.google.com/resource-manager/docs/tags/tags-overview. + properties: + key: + description: |- + key is required and is the short name of the pre-existing Resource Manager TagKey. + A key is 1-63 characters, begins and ends with + an ASCII alphanumeric character, and may contain letters, digits, '.', + '_', or '-' between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: key must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + '.', '_' or '-' + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$') + parentID: + description: |- + parentID is the ID of the project or organization that defines the TagKey. + When omitted, the HostedCluster's GCP project is used. + An explicit value must be the HostedCluster's GCP project ID or a numeric + organization ID. A numeric value is treated as an organization ID; project + numbers are not supported. + An organization ID consists of 1 to 32 decimal digits and must not start with 0. + A project ID is 6 to 30 characters long, contains only lowercase letters, digits + and hyphens, starts with a letter and must not end with a hyphen. + maxLength: 32 + minLength: 1 + type: string + x-kubernetes-validations: + - message: parentID must be a numeric organization ID + or a valid GCP project ID + rule: self.matches('(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)') + value: + description: |- + value is required and is the short name of the pre-existing Resource Manager TagValue for + key. Exactly one value for a TagKey can be attached to a resource. A value + is 1-63 characters, begins and ends with an ASCII alphanumeric character, + and may contain ASCII letters and digits, `_-.@%=+:,*#&()[]{}`, and + spaces between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: value must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + spaces or the characters _-.@%=+:,*#&()[]{} + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\\[\\]{}\\- + ]{0,61}[a-zA-Z0-9])?$') + required: + - key + - value + type: object + maxItems: 50 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: resourceTags are immutable + rule: self == oldSelf + - message: resourceTags keys must be unique across all parents + rule: self.all(t, self.exists_one(u, u.key == t.key)) workloadIdentity: description: |- workloadIdentity configures Workload Identity Federation for the cluster. @@ -8710,6 +8808,7 @@ spec: that manages GCS storage for the internal container image registry. This GSA requires the following IAM roles: - roles/storage.admin (Storage Admin - for creating and managing GCS buckets and objects) + - roles/resourcemanager.tagUser (Tag User - for applying resource tags to the bucket) See cmd/infra/gcp/iam-bindings.json for the authoritative role definitions. Format: service-account-name@project-id.iam.gserviceaccount.com @@ -8844,6 +8943,15 @@ spec: - message: network service account must belong to the same project rule: self.workloadIdentity.serviceAccountsEmails.network.endsWith('@' + self.project + '.iam.gserviceaccount.com') + - fieldPath: .resourceTags + message: resourceTags may only be configured during installation + rule: has(self.resourceTags) == has(oldSelf.resourceTags) + - fieldPath: .resourceTags + message: resourceTags parentID must be an organization ID or + the GCP platform project + rule: '!has(self.resourceTags) || self.resourceTags.all(t, !has(t.parentID) + || t.parentID.matches(''^[1-9][0-9]{0,31}$'') || t.parentID + == self.project)' ibmcloud: description: ibmcloud defines IBMCloud specific settings for components properties: @@ -9866,6 +9974,10 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: resourceTags may only be configured during installation + rule: (has(self.gcp) && has(self.gcp.resourceTags)) == (has(oldSelf.gcp) + && has(oldSelf.gcp.resourceTags)) pullSecret: description: |- pullSecret is a local reference to a Secret that must have a ".dockerconfigjson" key whose content must be a valid Openshift pull secret JSON. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml index 6ffb693347ce..2b5aba8eab09 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml @@ -7745,6 +7745,104 @@ spec: x-kubernetes-list-map-keys: - key x-kubernetes-list-type: map + resourceTags: + description: |- + resourceTags are pre-existing Google Cloud Resource Manager tags to apply to + GCP resources created for the hosted cluster. + Currently, tags are applied to newly provisioned persistent disks and, when the + image registry is enabled, to its storage bucket. Other GCP resources, such as + worker VMs and their boot disks, are not tagged. + The set of tagged resources is subject to change over time. + + When omitted, no tags are applied through this setting. When specified, + the list must contain 1 to 50 entries, and each short key must be unique + across the list, even when the tags have different parents. A tag's + parentID defaults to the HostedCluster's GCP project if omitted. + Google Cloud permits at most 50 directly attached tags per resource, + including tags attached by other actors. Coordinate with external tagging + automation to leave sufficient capacity; admission does not reserve tag slots. + + The referenced tags must already exist, and the storage and imageRegistry + service accounts must have permission to use them. Neither is checked when the + HostedCluster is created. When a tag cannot be applied, provisioning of the + affected disk or bucket can fail, and the error is reported in the persistent + volume claim's events or the image registry operator's status. + + resourceTags can only be set when the HostedCluster is created. It cannot be + added, removed or changed afterwards. If the configured list must change, + the HostedCluster must be recreated. + Resource tags require a control plane operator image that supports GCP resource + tag propagation. When that capability is unavailable, reconciliation fails + with a message identifying resourceTags on the ReconciliationSucceeded + condition, and dependent component deployments are blocked until a + supporting control plane operator image is configured. + items: + description: |- + GCPResourceTag selects a pre-existing Google Cloud Resource Manager tag by + its parent, short key, and short value. The tag must be defined in the + HostedCluster project or an organization before the cluster is created. + See https://cloud.google.com/resource-manager/docs/tags/tags-overview. + properties: + key: + description: |- + key is required and is the short name of the pre-existing Resource Manager TagKey. + A key is 1-63 characters, begins and ends with + an ASCII alphanumeric character, and may contain letters, digits, '.', + '_', or '-' between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: key must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + '.', '_' or '-' + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$') + parentID: + description: |- + parentID is the ID of the project or organization that defines the TagKey. + When omitted, the HostedCluster's GCP project is used. + An explicit value must be the HostedCluster's GCP project ID or a numeric + organization ID. A numeric value is treated as an organization ID; project + numbers are not supported. + An organization ID consists of 1 to 32 decimal digits and must not start with 0. + A project ID is 6 to 30 characters long, contains only lowercase letters, digits + and hyphens, starts with a letter and must not end with a hyphen. + maxLength: 32 + minLength: 1 + type: string + x-kubernetes-validations: + - message: parentID must be a numeric organization ID + or a valid GCP project ID + rule: self.matches('(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)') + value: + description: |- + value is required and is the short name of the pre-existing Resource Manager TagValue for + key. Exactly one value for a TagKey can be attached to a resource. A value + is 1-63 characters, begins and ends with an ASCII alphanumeric character, + and may contain ASCII letters and digits, `_-.@%=+:,*#&()[]{}`, and + spaces between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: value must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + spaces or the characters _-.@%=+:,*#&()[]{} + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\\[\\]{}\\- + ]{0,61}[a-zA-Z0-9])?$') + required: + - key + - value + type: object + maxItems: 50 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: resourceTags are immutable + rule: self == oldSelf + - message: resourceTags keys must be unique across all parents + rule: self.all(t, self.exists_one(u, u.key == t.key)) workloadIdentity: description: |- workloadIdentity configures Workload Identity Federation for the cluster. @@ -7885,6 +7983,7 @@ spec: that manages GCS storage for the internal container image registry. This GSA requires the following IAM roles: - roles/storage.admin (Storage Admin - for creating and managing GCS buckets and objects) + - roles/resourcemanager.tagUser (Tag User - for applying resource tags to the bucket) See cmd/infra/gcp/iam-bindings.json for the authoritative role definitions. Format: service-account-name@project-id.iam.gserviceaccount.com @@ -8019,6 +8118,15 @@ spec: - message: network service account must belong to the same project rule: self.workloadIdentity.serviceAccountsEmails.network.endsWith('@' + self.project + '.iam.gserviceaccount.com') + - fieldPath: .resourceTags + message: resourceTags may only be configured during installation + rule: has(self.resourceTags) == has(oldSelf.resourceTags) + - fieldPath: .resourceTags + message: resourceTags parentID must be an organization ID or + the GCP platform project + rule: '!has(self.resourceTags) || self.resourceTags.all(t, !has(t.parentID) + || t.parentID.matches(''^[1-9][0-9]{0,31}$'') || t.parentID + == self.project)' ibmcloud: description: ibmcloud defines IBMCloud specific settings for components properties: @@ -9041,6 +9149,10 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: resourceTags may only be configured during installation + rule: (has(self.gcp) && has(self.gcp.resourceTags)) == (has(oldSelf.gcp) + && has(oldSelf.gcp.resourceTags)) pullSecret: description: |- pullSecret is a local reference to a Secret that must have a ".dockerconfigjson" key whose content must be a valid Openshift pull secret JSON. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml index 80d16ebb036c..1e6bc6170c57 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml @@ -8447,6 +8447,104 @@ spec: x-kubernetes-list-map-keys: - key x-kubernetes-list-type: map + resourceTags: + description: |- + resourceTags are pre-existing Google Cloud Resource Manager tags to apply to + GCP resources created for the hosted cluster. + Currently, tags are applied to newly provisioned persistent disks and, when the + image registry is enabled, to its storage bucket. Other GCP resources, such as + worker VMs and their boot disks, are not tagged. + The set of tagged resources is subject to change over time. + + When omitted, no tags are applied through this setting. When specified, + the list must contain 1 to 50 entries, and each short key must be unique + across the list, even when the tags have different parents. A tag's + parentID defaults to the HostedCluster's GCP project if omitted. + Google Cloud permits at most 50 directly attached tags per resource, + including tags attached by other actors. Coordinate with external tagging + automation to leave sufficient capacity; admission does not reserve tag slots. + + The referenced tags must already exist, and the storage and imageRegistry + service accounts must have permission to use them. Neither is checked when the + HostedCluster is created. When a tag cannot be applied, provisioning of the + affected disk or bucket can fail, and the error is reported in the persistent + volume claim's events or the image registry operator's status. + + resourceTags can only be set when the HostedCluster is created. It cannot be + added, removed or changed afterwards. If the configured list must change, + the HostedCluster must be recreated. + Resource tags require a control plane operator image that supports GCP resource + tag propagation. When that capability is unavailable, reconciliation fails + with a message identifying resourceTags on the ReconciliationSucceeded + condition, and dependent component deployments are blocked until a + supporting control plane operator image is configured. + items: + description: |- + GCPResourceTag selects a pre-existing Google Cloud Resource Manager tag by + its parent, short key, and short value. The tag must be defined in the + HostedCluster project or an organization before the cluster is created. + See https://cloud.google.com/resource-manager/docs/tags/tags-overview. + properties: + key: + description: |- + key is required and is the short name of the pre-existing Resource Manager TagKey. + A key is 1-63 characters, begins and ends with + an ASCII alphanumeric character, and may contain letters, digits, '.', + '_', or '-' between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: key must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + '.', '_' or '-' + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$') + parentID: + description: |- + parentID is the ID of the project or organization that defines the TagKey. + When omitted, the HostedCluster's GCP project is used. + An explicit value must be the HostedCluster's GCP project ID or a numeric + organization ID. A numeric value is treated as an organization ID; project + numbers are not supported. + An organization ID consists of 1 to 32 decimal digits and must not start with 0. + A project ID is 6 to 30 characters long, contains only lowercase letters, digits + and hyphens, starts with a letter and must not end with a hyphen. + maxLength: 32 + minLength: 1 + type: string + x-kubernetes-validations: + - message: parentID must be a numeric organization ID + or a valid GCP project ID + rule: self.matches('(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)') + value: + description: |- + value is required and is the short name of the pre-existing Resource Manager TagValue for + key. Exactly one value for a TagKey can be attached to a resource. A value + is 1-63 characters, begins and ends with an ASCII alphanumeric character, + and may contain ASCII letters and digits, `_-.@%=+:,*#&()[]{}`, and + spaces between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: value must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + spaces or the characters _-.@%=+:,*#&()[]{} + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\\[\\]{}\\- + ]{0,61}[a-zA-Z0-9])?$') + required: + - key + - value + type: object + maxItems: 50 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: resourceTags are immutable + rule: self == oldSelf + - message: resourceTags keys must be unique across all parents + rule: self.all(t, self.exists_one(u, u.key == t.key)) workloadIdentity: description: |- workloadIdentity configures Workload Identity Federation for the cluster. @@ -8587,6 +8685,7 @@ spec: that manages GCS storage for the internal container image registry. This GSA requires the following IAM roles: - roles/storage.admin (Storage Admin - for creating and managing GCS buckets and objects) + - roles/resourcemanager.tagUser (Tag User - for applying resource tags to the bucket) See cmd/infra/gcp/iam-bindings.json for the authoritative role definitions. Format: service-account-name@project-id.iam.gserviceaccount.com @@ -8721,6 +8820,15 @@ spec: - message: network service account must belong to the same project rule: self.workloadIdentity.serviceAccountsEmails.network.endsWith('@' + self.project + '.iam.gserviceaccount.com') + - fieldPath: .resourceTags + message: resourceTags may only be configured during installation + rule: has(self.resourceTags) == has(oldSelf.resourceTags) + - fieldPath: .resourceTags + message: resourceTags parentID must be an organization ID or + the GCP platform project + rule: '!has(self.resourceTags) || self.resourceTags.all(t, !has(t.parentID) + || t.parentID.matches(''^[1-9][0-9]{0,31}$'') || t.parentID + == self.project)' ibmcloud: description: ibmcloud defines IBMCloud specific settings for components properties: @@ -9743,6 +9851,10 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: resourceTags may only be configured during installation + rule: (has(self.gcp) && has(self.gcp.resourceTags)) == (has(oldSelf.gcp) + && has(oldSelf.gcp.resourceTags)) pullSecret: description: pullSecret is a reference to a secret containing the pull secret for the hosted control plane. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml index 797e8cdad415..0c07831c6dac 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml @@ -7622,6 +7622,104 @@ spec: x-kubernetes-list-map-keys: - key x-kubernetes-list-type: map + resourceTags: + description: |- + resourceTags are pre-existing Google Cloud Resource Manager tags to apply to + GCP resources created for the hosted cluster. + Currently, tags are applied to newly provisioned persistent disks and, when the + image registry is enabled, to its storage bucket. Other GCP resources, such as + worker VMs and their boot disks, are not tagged. + The set of tagged resources is subject to change over time. + + When omitted, no tags are applied through this setting. When specified, + the list must contain 1 to 50 entries, and each short key must be unique + across the list, even when the tags have different parents. A tag's + parentID defaults to the HostedCluster's GCP project if omitted. + Google Cloud permits at most 50 directly attached tags per resource, + including tags attached by other actors. Coordinate with external tagging + automation to leave sufficient capacity; admission does not reserve tag slots. + + The referenced tags must already exist, and the storage and imageRegistry + service accounts must have permission to use them. Neither is checked when the + HostedCluster is created. When a tag cannot be applied, provisioning of the + affected disk or bucket can fail, and the error is reported in the persistent + volume claim's events or the image registry operator's status. + + resourceTags can only be set when the HostedCluster is created. It cannot be + added, removed or changed afterwards. If the configured list must change, + the HostedCluster must be recreated. + Resource tags require a control plane operator image that supports GCP resource + tag propagation. When that capability is unavailable, reconciliation fails + with a message identifying resourceTags on the ReconciliationSucceeded + condition, and dependent component deployments are blocked until a + supporting control plane operator image is configured. + items: + description: |- + GCPResourceTag selects a pre-existing Google Cloud Resource Manager tag by + its parent, short key, and short value. The tag must be defined in the + HostedCluster project or an organization before the cluster is created. + See https://cloud.google.com/resource-manager/docs/tags/tags-overview. + properties: + key: + description: |- + key is required and is the short name of the pre-existing Resource Manager TagKey. + A key is 1-63 characters, begins and ends with + an ASCII alphanumeric character, and may contain letters, digits, '.', + '_', or '-' between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: key must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + '.', '_' or '-' + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$') + parentID: + description: |- + parentID is the ID of the project or organization that defines the TagKey. + When omitted, the HostedCluster's GCP project is used. + An explicit value must be the HostedCluster's GCP project ID or a numeric + organization ID. A numeric value is treated as an organization ID; project + numbers are not supported. + An organization ID consists of 1 to 32 decimal digits and must not start with 0. + A project ID is 6 to 30 characters long, contains only lowercase letters, digits + and hyphens, starts with a letter and must not end with a hyphen. + maxLength: 32 + minLength: 1 + type: string + x-kubernetes-validations: + - message: parentID must be a numeric organization ID + or a valid GCP project ID + rule: self.matches('(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)') + value: + description: |- + value is required and is the short name of the pre-existing Resource Manager TagValue for + key. Exactly one value for a TagKey can be attached to a resource. A value + is 1-63 characters, begins and ends with an ASCII alphanumeric character, + and may contain ASCII letters and digits, `_-.@%=+:,*#&()[]{}`, and + spaces between them. + maxLength: 63 + minLength: 1 + type: string + x-kubernetes-validations: + - message: value must start and end with an ASCII letter + or digit and contain only ASCII letters, digits, + spaces or the characters _-.@%=+:,*#&()[]{} + rule: self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\\[\\]{}\\- + ]{0,61}[a-zA-Z0-9])?$') + required: + - key + - value + type: object + maxItems: 50 + minItems: 1 + type: array + x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: resourceTags are immutable + rule: self == oldSelf + - message: resourceTags keys must be unique across all parents + rule: self.all(t, self.exists_one(u, u.key == t.key)) workloadIdentity: description: |- workloadIdentity configures Workload Identity Federation for the cluster. @@ -7762,6 +7860,7 @@ spec: that manages GCS storage for the internal container image registry. This GSA requires the following IAM roles: - roles/storage.admin (Storage Admin - for creating and managing GCS buckets and objects) + - roles/resourcemanager.tagUser (Tag User - for applying resource tags to the bucket) See cmd/infra/gcp/iam-bindings.json for the authoritative role definitions. Format: service-account-name@project-id.iam.gserviceaccount.com @@ -7896,6 +7995,15 @@ spec: - message: network service account must belong to the same project rule: self.workloadIdentity.serviceAccountsEmails.network.endsWith('@' + self.project + '.iam.gserviceaccount.com') + - fieldPath: .resourceTags + message: resourceTags may only be configured during installation + rule: has(self.resourceTags) == has(oldSelf.resourceTags) + - fieldPath: .resourceTags + message: resourceTags parentID must be an organization ID or + the GCP platform project + rule: '!has(self.resourceTags) || self.resourceTags.all(t, !has(t.parentID) + || t.parentID.matches(''^[1-9][0-9]{0,31}$'') || t.parentID + == self.project)' ibmcloud: description: ibmcloud defines IBMCloud specific settings for components properties: @@ -8918,6 +9026,10 @@ spec: required: - type type: object + x-kubernetes-validations: + - message: resourceTags may only be configured during installation + rule: (has(self.gcp) && has(self.gcp.resourceTags)) == (has(oldSelf.gcp) + && has(oldSelf.gcp.resourceTags)) pullSecret: description: pullSecret is a reference to a secret containing the pull secret for the hosted control plane. diff --git a/docs/content/how-to/gcp-platform.md b/docs/content/how-to/gcp-platform.md index 45665e38d74b..d259e92077da 100644 --- a/docs/content/how-to/gcp-platform.md +++ b/docs/content/how-to/gcp-platform.md @@ -96,6 +96,120 @@ key is later removed from the HCP, HyperShift removes it from the Service annotation; it does not restore the Service's former value. Labels whose keys are not managed by HyperShift are preserved. +### Resource Manager Tags + +Resource Manager tags are distinct from resource labels. Define the TagKeys and +TagValues in the HostedCluster's GCP project or an organization before +installation, then configure their short names. If `parentID` is omitted, the +HostedCluster project is used; for an organization-defined tag, set `parentID` +to its numeric organization ID: + +```yaml +spec: + platform: + gcp: + resourceTags: + - key: environment + value: production + - parentID: "123456789012" + key: cost-center + value: shared +``` + +An explicit `parentID` must be the HostedCluster project ID or a numeric +organization ID. A different project ID is not accepted. HyperShift does not +check whether the referenced tag exists or whether the guest identities have +permission to use it at admission time. + +Google-managed system tags, such as +`google/system:sdp-data-sensitivity` and `google:AppHub/environment`, +cannot be configured through `resourceTags`. Their Google-managed namespaces +are not accepted by `parentID`. Google services may independently attach +system tags to resources. See [Google Cloud system +tags](https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing#system_tags). + +Keys and values must each be 1–63 characters long and begin and end with an +ASCII letter or digit. Keys may also contain `.`, `_`, and `-`; values accept +additional punctuation and spaces. When omitted, no tags are applied through +this setting. If present, the list must contain 1–50 entries. **Each short key +must be unique across the entire list, even when the tags have different +parents.** Thus an organization and project tag with the same short key cannot +both be configured with the current guest Infrastructure API. The HostedCluster +API enforces the accepted characters. + +The configured tags currently apply to these resources: + +| Tagged resource | When tagging is attempted | +| --- | --- | +| GCP PD CSI persistent disks for guest PersistentVolumes | When a new disk is provisioned | +| Image registry bucket | During bucket setup, when the image registry is enabled | + +They are **not** applied by this setting to worker VMs or their boot disks, +Private Service Connect forwarding rules or IP addresses, firewall rules, DNS +resources, or other GCP resources created by HyperShift or CAPG. Broader +management-side resource tagging is separate work. HyperShift does not create +TagKeys or TagValues. + +Google Cloud permits at most 50 directly attached Resource Manager tags per +resource. Organization-defined and project-defined tags share this limit when +attached to the same resource. Tags attached by administrators or external +automation also consume these slots. + +Before configuring `resourceTags`, account for any other direct bindings your +organization applies to the target disks and registry bucket. There is no fixed +number of reserved slots: the required headroom depends on your tagging +automation. Accepting a list at admission does not reserve capacity in Google +Cloud or prevent another actor from attaching tags. + +See [Google Cloud tag +limits](https://cloud.google.com/resource-manager/docs/limits#tag_limits). + +HyperShift passes these tags to the guest cluster for the two components above. +Resource tags require OpenShift 5.1 or later, or another control plane operator +image that supports GCP resource tag propagation. The selected image capability +label is authoritative: when it is absent, `ReconciliationSucceeded` is +`False` with reason `ReconciliationError`. Its message identifies +`spec.platform.gcp.resourceTags` as requiring a supporting CPO image and lists +blocked operations. Dependent component deployments are blocked, while core +HostedControlPlane reconciliation and independent synchronization can continue. +This check does not change `ValidConfiguration`. + +Configure tags when creating the HostedCluster: `resourceTags` cannot be added, +removed, or changed afterward. The guest components that +attach tags (the storage and image registry service-account identities) require +Google Cloud Tag User on the tag value and target resource, plus the applicable +resource-specific TagBinding permissions. HyperShift does not currently report +a dedicated condition for missing or inaccessible TagKeys or TagValues, or for +missing tag permissions. The `ValidGCPCredentials` and `ValidGCPWorkloadIdentity` conditions +do not validate the guest storage and image-registry identities' tag access. +Such failures do not block HyperShift's HostedCluster reconciliation, but they +can prevent the guest components from creating a tagged registry bucket or +persistent disk. Check the guest image-registry operator's status and logs, or +the GCP PD CSI controller logs and affected PVC events, for runtime errors. +See [Create GCP IAM Resources](gcp/create-gcp-iam.md#resource-tag-permissions) +for the required grants and how to apply them to existing clusters. + +#### Recovering from tag application failures + +Check the guest image-registry operator status and logs, or the PD CSI +controller logs and affected PVC events, to identify the failure. + +For the registry bucket, tags are attached individually after bucket creation. +Some bindings can succeed while others fail; successfully attached tags are +not rolled back. If external bindings exhaust the tag limit, coordinate with +their owner to remove unnecessary bindings and prevent automation from +immediately recreating them. The registry operator retries failed tagging +operations during reconciliation. + +For missing tags or insufficient permissions, correct the referenced tag +definitions or IAM grants. These corrections do not require changing the +HostedCluster tag list. + +The HostedCluster `resourceTags` list cannot be edited after creation. If +recovery requires changing that list rather than correcting external bindings, +tag definitions, or permissions, recreate the HostedCluster with the corrected +configuration. + ## CAPG Integration ### Controller Deployment diff --git a/docs/content/how-to/gcp/configure-image-registry.md b/docs/content/how-to/gcp/configure-image-registry.md index a5ed4e8e64ac..fd67a92af373 100644 --- a/docs/content/how-to/gcp/configure-image-registry.md +++ b/docs/content/how-to/gcp/configure-image-registry.md @@ -184,6 +184,18 @@ gcloud projects add-iam-policy-binding \ --role="roles/storage.admin" ``` +### Resource Tag Binding Fails + +If the HostedCluster specifies `resourceTags`, the registry operator also needs +`roles/resourcemanager.tagUser` to attach them to its bucket. Check the guest +registry operator status and logs for tag-binding failures, and verify that +the `image-registry` GSA has Tag User on the hosted cluster project and on any +organization-defined TagValues. Service accounts created with an earlier +version of `hypershift create iam gcp` do not have Tag User. See [Resource tag +permissions](create-gcp-iam.md#resource-tag-permissions) for setup and +remediation. HyperShift does not currently expose a dedicated HostedCluster +condition for these failures. + ### WIF Authentication Errors If the registry operator logs show token exchange errors (e.g., `invalid_grant` or `audience mismatch`): diff --git a/docs/content/how-to/gcp/create-gcp-iam.md b/docs/content/how-to/gcp/create-gcp-iam.md index 48d1b383fe21..a774b213a206 100644 --- a/docs/content/how-to/gcp/create-gcp-iam.md +++ b/docs/content/how-to/gcp/create-gcp-iam.md @@ -56,8 +56,8 @@ The `hypershift create iam gcp` command creates WIF resources in the hosted clus - `ctrlplane-op` — Control Plane Operator (DNS admin, network admin, security admin, compute viewer). The `roles/compute.securityAdmin` role lets CPO reconcile the worker firewall rule (`-internal-cluster`). - `nodepool` — CAPG controller (compute instance admin, network admin) - `cloud-controller` — Cloud Controller Manager (load balancer admin, security admin, compute viewer) - - `storage` — GCP PD CSI Driver (storage admin, instance admin) - - `image-registry` — Image Registry Operator (storage admin) + - `storage` — GCP PD CSI Driver (storage admin, instance admin, Tag User) + - `image-registry` — Image Registry Operator (storage admin, Tag User) - `cloud-network` — Cloud Network Config Controller (instance admin, network user) ```bash @@ -117,6 +117,38 @@ The command outputs JSON with the WIF configuration: Save this output — you will need the project number, pool/provider IDs, and service account emails when creating the hosted cluster. +### Resource tag permissions + +When `resourceTags` is configured on a GCP HostedCluster, the guest GCP PD CSI +driver uses `serviceAccountsEmails.storage` to tag new persistent disks, and the +guest image registry operator uses `serviceAccountsEmails.imageRegistry` to tag +its bucket. `hypershift create iam gcp` grants both Google service accounts +`roles/resourcemanager.tagUser` on the hosted cluster project, in addition to +their storage roles. The Cloud Resource Manager API must also be enabled in the +project containing the resources to be tagged. See [Google Cloud tag +requirements](https://cloud.google.com/resource-manager/docs/tags/tags-creating-and-managing). + +The IAM template is applied only when `hypershift create iam gcp` runs. If the +service accounts were created with an earlier version of the command, the +image-registry service account does not have Tag User. Re-run the command with +the same inputs, which adds missing role bindings to existing service accounts, +or have an administrator add the project-level role binding. + +For an **organization-defined TagValue**, a project-level grant alone is not +enough. An organization administrator must also grant Tag User on each +configured TagValue to **both** service accounts. For example, repeat this +command with the storage and image-registry GSA emails: + +```bash +gcloud resource-manager tags values add-iam-policy-binding // \ + --member="serviceAccount:" \ + --role="roles/resourcemanager.tagUser" +``` + +HyperShift does not create TagKeys, TagValues, or organization-level IAM +bindings. See the [gcloud TagValue IAM +reference](https://cloud.google.com/sdk/gcloud/reference/resource-manager/tags/values/add-iam-policy-binding). + ## Destroy IAM Resources To clean up WIF resources: diff --git a/docs/content/reference/aggregated-docs.md b/docs/content/reference/aggregated-docs.md index 26646c2c6767..63cd79c6c242 100644 --- a/docs/content/reference/aggregated-docs.md +++ b/docs/content/reference/aggregated-docs.md @@ -26759,6 +26759,120 @@ key is later removed from the HCP, HyperShift removes it from the Service annotation; it does not restore the Service's former value. Labels whose keys are not managed by HyperShift are preserved. +### Resource Manager Tags + +Resource Manager tags are distinct from resource labels. Define the TagKeys and +TagValues in the HostedCluster's GCP project or an organization before +installation, then configure their short names. If `parentID` is omitted, the +HostedCluster project is used; for an organization-defined tag, set `parentID` +to its numeric organization ID: + +```yaml +spec: + platform: + gcp: + resourceTags: + - key: environment + value: production + - parentID: "123456789012" + key: cost-center + value: shared +``` + +An explicit `parentID` must be the HostedCluster project ID or a numeric +organization ID. A different project ID is not accepted. HyperShift does not +check whether the referenced tag exists or whether the guest identities have +permission to use it at admission time. + +Google-managed system tags, such as +`google/system:sdp-data-sensitivity` and `google:AppHub/environment`, +cannot be configured through `resourceTags`. Their Google-managed namespaces +are not accepted by `parentID`. Google services may independently attach +system tags to resources. See Google Cloud system +tags. + +Keys and values must each be 1–63 characters long and begin and end with an +ASCII letter or digit. Keys may also contain `.`, `_`, and `-`; values accept +additional punctuation and spaces. When omitted, no tags are applied through +this setting. If present, the list must contain 1–50 entries. **Each short key +must be unique across the entire list, even when the tags have different +parents.** Thus an organization and project tag with the same short key cannot +both be configured with the current guest Infrastructure API. The HostedCluster +API enforces the accepted characters. + +The configured tags currently apply to these resources: + +| Tagged resource | When tagging is attempted | +| --- | --- | +| GCP PD CSI persistent disks for guest PersistentVolumes | When a new disk is provisioned | +| Image registry bucket | During bucket setup, when the image registry is enabled | + +They are **not** applied by this setting to worker VMs or their boot disks, +Private Service Connect forwarding rules or IP addresses, firewall rules, DNS +resources, or other GCP resources created by HyperShift or CAPG. Broader +management-side resource tagging is separate work. HyperShift does not create +TagKeys or TagValues. + +Google Cloud permits at most 50 directly attached Resource Manager tags per +resource. Organization-defined and project-defined tags share this limit when +attached to the same resource. Tags attached by administrators or external +automation also consume these slots. + +Before configuring `resourceTags`, account for any other direct bindings your +organization applies to the target disks and registry bucket. There is no fixed +number of reserved slots: the required headroom depends on your tagging +automation. Accepting a list at admission does not reserve capacity in Google +Cloud or prevent another actor from attaching tags. + +See Google Cloud tag +limits. + +HyperShift passes these tags to the guest cluster for the two components above. +Resource tags require OpenShift 5.1 or later, or another control plane operator +image that supports GCP resource tag propagation. The selected image capability +label is authoritative: when it is absent, `ReconciliationSucceeded` is +`False` with reason `ReconciliationError`. Its message identifies +`spec.platform.gcp.resourceTags` as requiring a supporting CPO image and lists +blocked operations. Dependent component deployments are blocked, while core +HostedControlPlane reconciliation and independent synchronization can continue. +This check does not change `ValidConfiguration`. + +Configure tags when creating the HostedCluster: `resourceTags` cannot be added, +removed, or changed afterward. The guest components that +attach tags (the storage and image registry service-account identities) require +Google Cloud Tag User on the tag value and target resource, plus the applicable +resource-specific TagBinding permissions. HyperShift does not currently report +a dedicated condition for missing or inaccessible TagKeys or TagValues, or for +missing tag permissions. The `ValidGCPCredentials` and `ValidGCPWorkloadIdentity` conditions +do not validate the guest storage and image-registry identities' tag access. +Such failures do not block HyperShift's HostedCluster reconciliation, but they +can prevent the guest components from creating a tagged registry bucket or +persistent disk. Check the guest image-registry operator's status and logs, or +the GCP PD CSI controller logs and affected PVC events, for runtime errors. +See Create GCP IAM Resources +for the required grants and how to apply them to existing clusters. + +#### Recovering from tag application failures + +Check the guest image-registry operator status and logs, or the PD CSI +controller logs and affected PVC events, to identify the failure. + +For the registry bucket, tags are attached individually after bucket creation. +Some bindings can succeed while others fail; successfully attached tags are +not rolled back. If external bindings exhaust the tag limit, coordinate with +their owner to remove unnecessary bindings and prevent automation from +immediately recreating them. The registry operator retries failed tagging +operations during reconciliation. + +For missing tags or insufficient permissions, correct the referenced tag +definitions or IAM grants. These corrections do not require changing the +HostedCluster tag list. + +The HostedCluster `resourceTags` list cannot be edited after creation. If +recovery requires changing that list rather than correcting external bindings, +tag definitions, or permissions, recreate the HostedCluster with the corrected +configuration. + ## CAPG Integration ### Controller Deployment @@ -27093,6 +27207,18 @@ gcloud projects add-iam-policy-binding \ --role="roles/storage.admin" ``` +### Resource Tag Binding Fails + +If the HostedCluster specifies `resourceTags`, the registry operator also needs +`roles/resourcemanager.tagUser` to attach them to its bucket. Check the guest +registry operator status and logs for tag-binding failures, and verify that +the `image-registry` GSA has Tag User on the hosted cluster project and on any +organization-defined TagValues. Service accounts created with an earlier +version of `hypershift create iam gcp` do not have Tag User. See Resource tag +permissions for setup and +remediation. HyperShift does not currently expose a dedicated HostedCluster +condition for these failures. + ### WIF Authentication Errors If the registry operator logs show token exchange errors (e.g., `invalid_grant` or `audience mismatch`): @@ -27425,8 +27551,8 @@ The `hypershift create iam gcp` command creates WIF resources in the hosted clus - `ctrlplane-op` — Control Plane Operator (DNS admin, network admin, security admin, compute viewer). The `roles/compute.securityAdmin` role lets CPO reconcile the worker firewall rule (`-internal-cluster`). - `nodepool` — CAPG controller (compute instance admin, network admin) - `cloud-controller` — Cloud Controller Manager (load balancer admin, security admin, compute viewer) - - `storage` — GCP PD CSI Driver (storage admin, instance admin) - - `image-registry` — Image Registry Operator (storage admin) + - `storage` — GCP PD CSI Driver (storage admin, instance admin, Tag User) + - `image-registry` — Image Registry Operator (storage admin, Tag User) - `cloud-network` — Cloud Network Config Controller (instance admin, network user) ```bash @@ -27486,6 +27612,38 @@ The command outputs JSON with the WIF configuration: Save this output — you will need the project number, pool/provider IDs, and service account emails when creating the hosted cluster. +### Resource tag permissions + +When `resourceTags` is configured on a GCP HostedCluster, the guest GCP PD CSI +driver uses `serviceAccountsEmails.storage` to tag new persistent disks, and the +guest image registry operator uses `serviceAccountsEmails.imageRegistry` to tag +its bucket. `hypershift create iam gcp` grants both Google service accounts +`roles/resourcemanager.tagUser` on the hosted cluster project, in addition to +their storage roles. The Cloud Resource Manager API must also be enabled in the +project containing the resources to be tagged. See Google Cloud tag +requirements. + +The IAM template is applied only when `hypershift create iam gcp` runs. If the +service accounts were created with an earlier version of the command, the +image-registry service account does not have Tag User. Re-run the command with +the same inputs, which adds missing role bindings to existing service accounts, +or have an administrator add the project-level role binding. + +For an **organization-defined TagValue**, a project-level grant alone is not +enough. An organization administrator must also grant Tag User on each +configured TagValue to **both** service accounts. For example, repeat this +command with the storage and image-registry GSA emails: + +```bash +gcloud resource-manager tags values add-iam-policy-binding // \ + --member="serviceAccount:" \ + --role="roles/resourcemanager.tagUser" +``` + +HyperShift does not create TagKeys, TagValues, or organization-level IAM +bindings. See the gcloud TagValue IAM +reference. + ## Destroy IAM Resources To clean up WIF resources: @@ -51636,6 +51794,45 @@ For GCP labeling guidance, see +[]GCPResourceTag + + + + +(Optional) +

resourceTags are pre-existing Google Cloud Resource Manager tags to apply to +GCP resources created for the hosted cluster. +Currently, tags are applied to newly provisioned persistent disks and, when the +image registry is enabled, to its storage bucket. Other GCP resources, such as +worker VMs and their boot disks, are not tagged. +The set of tagged resources is subject to change over time.

+

When omitted, no tags are applied through this setting. When specified, +the list must contain 1 to 50 entries, and each short key must be unique +across the list, even when the tags have different parents. A tag’s +parentID defaults to the HostedCluster’s GCP project if omitted. +Google Cloud permits at most 50 directly attached tags per resource, +including tags attached by other actors. Coordinate with external tagging +automation to leave sufficient capacity; admission does not reserve tag slots.

+

The referenced tags must already exist, and the storage and imageRegistry +service accounts must have permission to use them. Neither is checked when the +HostedCluster is created. When a tag cannot be applied, provisioning of the +affected disk or bucket can fail, and the error is reported in the persistent +volume claim’s events or the image registry operator’s status.

+

resourceTags can only be set when the HostedCluster is created. It cannot be +added, removed or changed afterwards. If the configured list must change, +the HostedCluster must be recreated. +Resource tags require a control plane operator image that supports GCP resource +tag propagation. When that capability is unavailable, reconciliation fails +with a message identifying resourceTags on the ReconciliationSucceeded +condition, and dependent component deployments are blocked until a +supporting control plane operator image is configured.

+ + + + workloadIdentity,omitzero
@@ -51958,6 +52155,75 @@ See https://clo +###GCPResourceTag { #hypershift.openshift.io/v1beta1.GCPResourceTag } +

+(Appears on: +GCPPlatformSpec) +

+

+

GCPResourceTag selects a pre-existing Google Cloud Resource Manager tag by +its parent, short key, and short value. The tag must be defined in the +HostedCluster project or an organization before the cluster is created. +See https://cloud.google.com/resource-manager/docs/tags/tags-overview.

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+parentID
+ +string + +		 +(Optional) +

parentID is the ID of the project or organization that defines the TagKey. +When omitted, the HostedCluster’s GCP project is used. +An explicit value must be the HostedCluster’s GCP project ID or a numeric +organization ID. A numeric value is treated as an organization ID; project +numbers are not supported. +An organization ID consists of 1 to 32 decimal digits and must not start with 0. +A project ID is 6 to 30 characters long, contains only lowercase letters, digits +and hyphens, starts with a letter and must not end with a hyphen.

+
+key
+ +string + +		 +

key is required and is the short name of the pre-existing Resource Manager TagKey. +A key is 1-63 characters, begins and ends with +an ASCII alphanumeric character, and may contain letters, digits, ‘.’, +‘_’, or ‘-’ between them.

+
+value
+ +string + +		 +

value is required and is the short name of the pre-existing Resource Manager TagValue for +key. Exactly one value for a TagKey can be attached to a resource. A value +is 1-63 characters, begins and ends with an ASCII alphanumeric character, +and may contain ASCII letters and digits, _-.@%=+:,*#&()[]{}, and +spaces between them.

+
###GCPServiceAccountEmail { #hypershift.openshift.io/v1beta1.GCPServiceAccountEmail }

(Appears on: @@ -52094,6 +52360,7 @@ GCPServiceAccountEmail that manages GCS storage for the internal container image registry. This GSA requires the following IAM roles: - roles/storage.admin (Storage Admin - for creating and managing GCS buckets and objects) +- roles/resourcemanager.tagUser (Tag User - for applying resource tags to the bucket) See cmd/infra/gcp/iam-bindings.json for the authoritative role definitions. Format: service-account-name@project-id.iam.gserviceaccount.com

This is a user-provided value referencing a pre-created Google Service Account. diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index 8e9e9f34be69..ca5024edbfb2 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -8969,6 +8969,45 @@ For GCP labeling guidance, see +[]GCPResourceTag + + + + +(Optional) +

resourceTags are pre-existing Google Cloud Resource Manager tags to apply to +GCP resources created for the hosted cluster. +Currently, tags are applied to newly provisioned persistent disks and, when the +image registry is enabled, to its storage bucket. Other GCP resources, such as +worker VMs and their boot disks, are not tagged. +The set of tagged resources is subject to change over time.

+

When omitted, no tags are applied through this setting. When specified, +the list must contain 1 to 50 entries, and each short key must be unique +across the list, even when the tags have different parents. A tag’s +parentID defaults to the HostedCluster’s GCP project if omitted. +Google Cloud permits at most 50 directly attached tags per resource, +including tags attached by other actors. Coordinate with external tagging +automation to leave sufficient capacity; admission does not reserve tag slots.

+

The referenced tags must already exist, and the storage and imageRegistry +service accounts must have permission to use them. Neither is checked when the +HostedCluster is created. When a tag cannot be applied, provisioning of the +affected disk or bucket can fail, and the error is reported in the persistent +volume claim’s events or the image registry operator’s status.

+

resourceTags can only be set when the HostedCluster is created. It cannot be +added, removed or changed afterwards. If the configured list must change, +the HostedCluster must be recreated. +Resource tags require a control plane operator image that supports GCP resource +tag propagation. When that capability is unavailable, reconciliation fails +with a message identifying resourceTags on the ReconciliationSucceeded +condition, and dependent component deployments are blocked until a +supporting control plane operator image is configured.

+ + + + workloadIdentity,omitzero
@@ -9291,6 +9330,75 @@ See https://clo +###GCPResourceTag { #hypershift.openshift.io/v1beta1.GCPResourceTag } +

+(Appears on: +GCPPlatformSpec) +

+

+

GCPResourceTag selects a pre-existing Google Cloud Resource Manager tag by +its parent, short key, and short value. The tag must be defined in the +HostedCluster project or an organization before the cluster is created. +See https://cloud.google.com/resource-manager/docs/tags/tags-overview.

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+parentID
+ +string + +		 +(Optional) +

parentID is the ID of the project or organization that defines the TagKey. +When omitted, the HostedCluster’s GCP project is used. +An explicit value must be the HostedCluster’s GCP project ID or a numeric +organization ID. A numeric value is treated as an organization ID; project +numbers are not supported. +An organization ID consists of 1 to 32 decimal digits and must not start with 0. +A project ID is 6 to 30 characters long, contains only lowercase letters, digits +and hyphens, starts with a letter and must not end with a hyphen.

+
+key
+ +string + +		 +

key is required and is the short name of the pre-existing Resource Manager TagKey. +A key is 1-63 characters, begins and ends with +an ASCII alphanumeric character, and may contain letters, digits, ‘.’, +‘_’, or ‘-’ between them.

+
+value
+ +string + +		 +

value is required and is the short name of the pre-existing Resource Manager TagValue for +key. Exactly one value for a TagKey can be attached to a resource. A value +is 1-63 characters, begins and ends with an ASCII alphanumeric character, +and may contain ASCII letters and digits, _-.@%=+:,*#&()[]{}, and +spaces between them.

+
###GCPServiceAccountEmail { #hypershift.openshift.io/v1beta1.GCPServiceAccountEmail }

(Appears on: @@ -9427,6 +9535,7 @@ GCPServiceAccountEmail that manages GCS storage for the internal container image registry. This GSA requires the following IAM roles: - roles/storage.admin (Storage Admin - for creating and managing GCS buckets and objects) +- roles/resourcemanager.tagUser (Tag User - for applying resource tags to the bucket) See cmd/infra/gcp/iam-bindings.json for the authoritative role definitions. Format: service-account-name@project-id.iam.gserviceaccount.com

This is a user-provided value referencing a pre-created Google Service Account. diff --git a/hypershift-operator/controllers/hostedcluster/gcp_credentials.go b/hypershift-operator/controllers/hostedcluster/gcp_conditions.go similarity index 56% rename from hypershift-operator/controllers/hostedcluster/gcp_credentials.go rename to hypershift-operator/controllers/hostedcluster/gcp_conditions.go index 6056bdaefd49..68a02f378384 100644 --- a/hypershift-operator/controllers/hostedcluster/gcp_credentials.go +++ b/hypershift-operator/controllers/hostedcluster/gcp_conditions.go @@ -23,3 +23,17 @@ func (r *HostedClusterReconciler) reconcileGCPCredentialConditions(ctx context.C } return nil } + +// validateGCPResourceTagsSupport checks whether requested tags can be propagated +// by the selected CPO image, without changing the HostedCluster. Image-resolution +// failures are reported separately, so unresolved labels are not treated as a +// missing capability. +func validateGCPResourceTagsSupport(hc *hyperv1.HostedCluster, labelsResolved bool, imageLabels map[string]string) error { + if hc.Spec.Platform.Type != hyperv1.GCPPlatform || hc.Spec.Platform.GCP == nil || len(hc.Spec.Platform.GCP.ResourceTags) == 0 || !labelsResolved { + return nil + } + if _, supported := imageLabels[controlPlaneOperatorSupportsGCPResourceTagsLabel]; !supported { + return fmt.Errorf("spec.platform.gcp.resourceTags requires a control plane operator image that supports GCP resource tag propagation") + } + return nil +} diff --git a/hypershift-operator/controllers/hostedcluster/gcp_credentials_test.go b/hypershift-operator/controllers/hostedcluster/gcp_conditions_test.go similarity index 80% rename from hypershift-operator/controllers/hostedcluster/gcp_credentials_test.go rename to hypershift-operator/controllers/hostedcluster/gcp_conditions_test.go index f80dea1d9f44..d138807b46e6 100644 --- a/hypershift-operator/controllers/hostedcluster/gcp_credentials_test.go +++ b/hypershift-operator/controllers/hostedcluster/gcp_conditions_test.go @@ -23,7 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" ) -func TestReconcileGCPCredentialConditions(t *testing.T) { +func TestReconcileGCPConditions(t *testing.T) { for _, tc := range []struct { name string hcVersion, hcpVersion string @@ -267,3 +267,52 @@ func TestReconcileGCPCredentialConditions(t *testing.T) { g.Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) } + +func TestValidateGCPResourceTagsSupport(t *testing.T) { + for _, tc := range []struct { + name string + tags, supported, unresolved, otherPlatform, missingConfig bool + invalidConfiguration bool + expectedError string + }{ + {name: "When tags are omitted, it should not require CPO support"}, + {name: "When the GCP configuration is absent, it should not require CPO support", missingConfig: true}, + {name: "When the platform is not GCP, it should not require GCP tag support", tags: true, otherPlatform: true}, + {name: "When the CPO supports tags, it should succeed", tags: true, supported: true}, + {name: "When the CPO does not support tags, it should identify the unsupported field", tags: true, expectedError: "spec.platform.gcp.resourceTags requires a control plane operator image that supports GCP resource tag propagation"}, + {name: "When labels are unresolved, it should defer to the image-resolution error", tags: true, unresolved: true}, + {name: "When unrelated configuration is invalid and tags are supported, it should leave the condition unchanged", tags: true, supported: true, invalidConfiguration: true}, + {name: "When unrelated configuration is invalid and tags are unsupported, it should leave the condition unchanged and return the tag error", tags: true, invalidConfiguration: true, expectedError: "spec.platform.gcp.resourceTags requires a control plane operator image that supports GCP resource tag propagation"}, + } { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + hc := &hyperv1.HostedCluster{ + Spec: hyperv1.HostedClusterSpec{Platform: hyperv1.PlatformSpec{Type: hyperv1.GCPPlatform, GCP: &hyperv1.GCPPlatformSpec{}}}, + } + if tc.tags { + hc.Spec.Platform.GCP.ResourceTags = []hyperv1.GCPResourceTag{{Key: "environment", Value: "production"}} + } + if tc.otherPlatform { + hc.Spec.Platform.Type = hyperv1.NonePlatform + } + if tc.missingConfig { + hc.Spec.Platform.GCP = nil + } + if tc.invalidConfiguration { + hc.Status.Conditions = []metav1.Condition{{Type: string(hyperv1.ValidHostedClusterConfiguration), Status: metav1.ConditionFalse, Reason: hyperv1.InvalidConfigurationReason, Message: "unrelated configuration error"}} + } + var labels map[string]string + if tc.supported { + labels = map[string]string{controlPlaneOperatorSupportsGCPResourceTagsLabel: "true"} + } + before := hc.DeepCopy() + err := validateGCPResourceTagsSupport(hc, !tc.unresolved, labels) + if tc.expectedError == "" { + g.Expect(err).NotTo(HaveOccurred()) + } else { + g.Expect(err).To(MatchError(tc.expectedError)) + } + g.Expect(hc).To(Equal(before), "validation must not mutate spec or conditions") + }) + } +} diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index 0f01306a49f5..2bae7a7b7ca6 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -138,6 +138,7 @@ const ( controlPlaneOperatorSubcommandsLabel = "io.openshift.hypershift.control-plane-operator-subcommands" controlPlaneOperatorSupportsKASCustomKubeconfigLabel = "io.openshift.hypershift.control-plane-operator-supports-kas-custom-kubeconfig" + controlPlaneOperatorSupportsGCPResourceTagsLabel = "io.openshift.hypershift.control-plane-operator-supports-gcp-resource-tags" controlPlaneOperatorAppliesManagementKASNetworkPolicyLabel = "io.openshift.hypershift.control-plane-operator-applies-management-kas-network-policy-label" controlPlanePKIOperatorSignsCSRsLabel = "io.openshift.hypershift.control-plane-pki-operator-signs-csrs" @@ -1467,6 +1468,7 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques var pullSecretBytes []byte controlPlaneOperatorImage := r.HypershiftOperatorImage var controlPlaneOperatorImageLabels map[string]string + controlPlaneOperatorImageLabelsResolved := false report.execute("CPOImageAndNamespace", critical, func() error { var err error @@ -1482,10 +1484,15 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques if err != nil { return fmt.Errorf("failed to get controlPlaneOperatorImageLabels: %w", err) } + controlPlaneOperatorImageLabelsResolved = true return r.reconcileControlPlaneNamespace(ctx, createOrUpdate, hcluster, controlPlaneNamespace, controlPlaneOperatorImageLabels) }) + report.execute("ValidateAndReconcileGCPTags", critical, func() error { + return validateGCPResourceTagsSupport(hcluster, controlPlaneOperatorImageLabelsResolved, controlPlaneOperatorImageLabels) + }) + cpoHasUtilities := false utilitiesImage := r.HypershiftOperatorImage if _, hasLabel := controlPlaneOperatorImageLabels[controlPlaneOperatorSubcommandsLabel]; hasLabel { @@ -4543,7 +4550,6 @@ func (r *HostedClusterReconciler) validateReleaseImage(ctx context.Context, hc * } minSupportedVersion := supportedversion.GetMinSupportedVersion(hc) - return supportedversion.IsValidReleaseVersion(&version, currentVersion, &supportedversion.LatestSupportedVersion, &minSupportedVersion, hc.Spec.Networking.NetworkType, hc.Spec.Platform.Type) } diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go index 3d39fbc5cd75..daaeb08ef395 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go @@ -51,6 +51,7 @@ import ( "github.com/openshift/hypershift/support/util/fakeimagemetadataprovider" configv1 "github.com/openshift/api/config/v1" + imagev1 "github.com/openshift/api/image/v1" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" @@ -69,6 +70,7 @@ import ( "k8s.io/utils/ptr" capiaws "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + capigcp "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" capibmv1 "sigs.k8s.io/cluster-api-provider-ibmcloud/api/v1beta2" capov1alpha1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha1" capiv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" @@ -8954,7 +8956,152 @@ func TestReconcileNonBlockingBehavior(t *testing.T) { expectedErrSubstring string // verifyResources asserts that specific resources were still created despite the failure. verifyResources func(t *testing.T, client crclient.Client, hcpNamespace, hcName string) + verifyRecovery func(t *testing.T, r *HostedClusterReconciler, hc *hyperv1.HostedCluster) }{ + { + name: "When GCP resource tags are unsupported, it should still sync secrets and create HCP but block deployments", + mutateHC: func(t *testing.T, hc *hyperv1.HostedCluster, client crclient.Client) { + hc.Spec.Platform = hyperv1.PlatformSpec{ + Type: hyperv1.GCPPlatform, + GCP: &hyperv1.GCPPlatformSpec{ + Project: "my-project-123", + Region: "us-central1", + ResourceTags: []hyperv1.GCPResourceTag{{Key: "environment", Value: "production"}}, + WorkloadIdentity: hyperv1.GCPWorkloadIdentityConfig{ + ProjectNumber: "123456789012", + PoolID: "my-wif-pool", + ProviderID: "my-wif-provider", + ServiceAccountsEmails: hyperv1.GCPServiceAccountsEmails{ + NodePool: "nodepool@my-project-123.iam.gserviceaccount.com", + ControlPlane: "controlplane@my-project-123.iam.gserviceaccount.com", + CloudController: "cloudcontroller@my-project-123.iam.gserviceaccount.com", + Storage: "storage@my-project-123.iam.gserviceaccount.com", + ImageRegistry: "imageregistry@my-project-123.iam.gserviceaccount.com", + Network: "network@my-project-123.iam.gserviceaccount.com", + }, + }, + }, + } + managementNetwork := &configv1.Network{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, + Spec: configv1.NetworkSpec{ + NetworkType: "OVNKubernetes", + ClusterNetwork: []configv1.ClusterNetworkEntry{{CIDR: "10.128.0.0/14", HostPrefix: 23}}, + ServiceNetwork: []string{"172.30.0.0/16"}, + }, + } + if err := client.Create(t.Context(), managementNetwork); err != nil { + t.Fatalf("failed to create management network: %v", err) + } + hc.Spec.SSHKey = corev1.LocalObjectReference{Name: "test-ssh-key"} + sshKey := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: hc.Namespace, Name: hc.Spec.SSHKey.Name}, + Data: map[string][]byte{"id_rsa.pub": []byte("ssh-rsa AAAA...")}, + } + if err := client.Create(t.Context(), sshKey); err != nil { + t.Fatalf("failed to create SSH key: %v", err) + } + }, + expectedErrSubstring: "critical error: spec.platform.gcp.resourceTags requires a control plane operator image", + verifyResources: func(t *testing.T, client crclient.Client, hcpNamespace, hcName string) { + g := NewWithT(t) + hcp := &hyperv1.HostedControlPlane{} + g.Expect(client.Get(t.Context(), crclient.ObjectKey{Namespace: hcpNamespace, Name: hcName}, hcp)).To(Succeed()) + g.Expect(hcp.Spec.Platform.GCP).NotTo(BeNil()) + g.Expect(hcp.Spec.Platform.GCP.ResourceTags).To(Equal([]hyperv1.GCPResourceTag{{Key: "environment", Value: "production"}})) + + sshKey := controlplaneoperator.SSHKey(hcpNamespace) + g.Expect(client.Get(t.Context(), crclient.ObjectKeyFromObject(sshKey), sshKey)).To(Succeed()) + g.Expect(sshKey.Data["id_rsa.pub"]).To(Equal([]byte("ssh-rsa AAAA..."))) + + deployment := &appsv1.Deployment{} + g.Expect(errors2.IsNotFound(client.Get(t.Context(), crclient.ObjectKey{Namespace: hcpNamespace, Name: "control-plane-operator"}, deployment))).To(BeTrue()) + + hc := &hyperv1.HostedCluster{} + g.Expect(client.Get(t.Context(), crclient.ObjectKey{Namespace: "test-ns", Name: hcName}, hc)).To(Succeed()) + condition := meta.FindStatusCondition(hc.Status.Conditions, string(hyperv1.ValidHostedClusterConfiguration)) + g.Expect(condition).NotTo(BeNil()) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(condition.Reason).To(Equal(hyperv1.AsExpectedReason)) + reconciled := meta.FindStatusCondition(hc.Status.Conditions, string(hyperv1.ReconciliationSucceeded)) + g.Expect(reconciled).NotTo(BeNil()) + g.Expect(reconciled.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(reconciled.Reason).To(Equal("ReconciliationError")) + g.Expect(reconciled.Message).To(ContainSubstring("critical error: spec.platform.gcp.resourceTags requires")) + g.Expect(reconciled.Message).To(ContainSubstring("blocked operations:")) + g.Expect(reconciled.Message).To(ContainSubstring("OperatorDeployments")) + }, + verifyRecovery: func(t *testing.T, r *HostedClusterReconciler, hc *hyperv1.HostedCluster) { + g := NewWithT(t) + reconcileAgain := func() error { + g.Expect(r.Client.Get(t.Context(), crclient.ObjectKeyFromObject(hc), hc)).To(Succeed()) + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: crclient.ObjectKeyFromObject(hc)}) + g.Expect(r.Client.Get(t.Context(), crclient.ObjectKeyFromObject(hc), hc)).To(Succeed()) + return err + } + // A previous failure must not short-circuit the next capability check. + g.Expect(reconcileAgain()).To(MatchError(ContainSubstring("critical error: spec.platform.gcp.resourceTags"))) + + // An image-resolution outage is reported without misdiagnosing tag support. + hc.Annotations[hyperv1.ControlPlaneOperatorImageLabelsAnnotation] = "malformed-label" + g.Expect(r.Client.Update(t.Context(), hc)).To(Succeed()) + g.Expect(reconcileAgain()).To(MatchError(ContainSubstring("failed to get controlPlaneOperatorImageLabels"))) + g.Expect(r.Client.Get(t.Context(), crclient.ObjectKeyFromObject(hc), hc)).To(Succeed()) + condition := meta.FindStatusCondition(hc.Status.Conditions, string(hyperv1.ValidHostedClusterConfiguration)) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(condition.Reason).To(Equal(hyperv1.AsExpectedReason)) + reconciled := meta.FindStatusCondition(hc.Status.Conditions, string(hyperv1.ReconciliationSucceeded)) + g.Expect(reconciled).NotTo(BeNil()) + g.Expect(reconciled.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(reconciled.Message).To(ContainSubstring("failed to get controlPlaneOperatorImageLabels")) + g.Expect(reconciled.Message).NotTo(ContainSubstring("resourceTags requires")) + + // A supporting image must recover without any special reset of status. + hc.Annotations[hyperv1.ControlPlaneOperatorImageLabelsAnnotation] = controlPlaneOperatorSupportsGCPResourceTagsLabel + "=true" + g.Expect(r.Client.Update(t.Context(), hc)).To(Succeed()) + g.Expect(reconcileAgain()).To(Succeed()) + g.Expect(r.Client.Get(t.Context(), crclient.ObjectKeyFromObject(hc), hc)).To(Succeed()) + condition = meta.FindStatusCondition(hc.Status.Conditions, string(hyperv1.ValidHostedClusterConfiguration)) + g.Expect(condition.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(condition.Reason).To(Equal(hyperv1.AsExpectedReason)) + reconciled = meta.FindStatusCondition(hc.Status.Conditions, string(hyperv1.ReconciliationSucceeded)) + g.Expect(reconciled.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(reconciled.Message).To(Equal("Reconciliation completed successfully")) + deployment := &appsv1.Deployment{} + g.Expect(r.Client.Get(t.Context(), crclient.ObjectKey{Namespace: hcpmanifests.HostedControlPlaneNamespace(hc.Namespace, hc.Name), Name: "control-plane-operator"}, deployment)).To(Succeed()) + }, + }, + { + name: "When GCP configuration is invalid, it should block at the configuration gate without a capability exception", + mutateHC: func(t *testing.T, hc *hyperv1.HostedCluster, client crclient.Client) { + hc.Spec.ClusterID = "not-a-uuid" + hc.Spec.Platform = hyperv1.PlatformSpec{ + Type: hyperv1.GCPPlatform, + GCP: &hyperv1.GCPPlatformSpec{ + Project: "my-project-123", Region: "us-central1", + ResourceTags: []hyperv1.GCPResourceTag{{Key: "environment", Value: "production"}}, + }, + } + }, + expectedErrSubstring: "configuration is invalid", + verifyResources: func(t *testing.T, client crclient.Client, hcpNamespace, hcName string) { + g := NewWithT(t) + hc := &hyperv1.HostedCluster{} + g.Expect(client.Get(t.Context(), crclient.ObjectKey{Namespace: "test-ns", Name: hcName}, hc)).To(Succeed()) + condition := meta.FindStatusCondition(hc.Status.Conditions, string(hyperv1.ValidHostedClusterConfiguration)) + g.Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(condition.Reason).To(Equal(hyperv1.InvalidConfigurationReason)) + g.Expect(condition.Message).To(ContainSubstring("cluster ID")) + reconciled := meta.FindStatusCondition(hc.Status.Conditions, string(hyperv1.ReconciliationSucceeded)) + g.Expect(reconciled.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(reconciled.Message).To(ContainSubstring("configuration is invalid")) + g.Expect(reconciled.Message).NotTo(ContainSubstring("resourceTags requires")) + hcp := &hyperv1.HostedControlPlane{} + g.Expect(errors2.IsNotFound(client.Get(t.Context(), crclient.ObjectKey{Namespace: hcpNamespace, Name: hcName}, hcp))).To(BeTrue()) + deployment := &appsv1.Deployment{} + g.Expect(errors2.IsNotFound(client.Get(t.Context(), crclient.ObjectKey{Namespace: hcpNamespace, Name: "control-plane-operator"}, deployment))).To(BeTrue()) + }, + }, { name: "When Phase 6 SSH key fails it should still create HCP and CPO deployment", mutateHC: func(t *testing.T, hc *hyperv1.HostedCluster, client crclient.Client) { @@ -9083,9 +9230,14 @@ func TestReconcileNonBlockingBehavior(t *testing.T) { mockCtrl := gomock.NewController(t) mockReleaseProvider := releaseinfo.NewMockProviderWithOpenShiftImageRegistryOverrides(mockCtrl) + releaseImage := testutils.InitReleaseImageOrDie("4.16.0") + releaseImage.ImageStream.Spec.Tags = append(releaseImage.ImageStream.Spec.Tags, imagev1.TagReference{ + Name: "gcp-cluster-api-controllers", + From: &corev1.ObjectReference{Name: "quay.io/openshift/origin-gcp-cluster-api-controllers:latest"}, + }) mockReleaseProvider.EXPECT(). Lookup(gomock.Any(), gomock.Any(), gomock.Any()). - Return(testutils.InitReleaseImageOrDie("4.16.0"), nil).AnyTimes() + Return(releaseImage, nil).AnyTimes() mockReleaseProvider.EXPECT().GetRegistryOverrides().Return(nil).AnyTimes() mockReleaseProvider.EXPECT().GetOpenShiftImageRegistryOverrides().Return(nil).AnyTimes() mockReleaseProvider.EXPECT().GetMirroredReleaseImage().Return("").AnyTimes() @@ -9141,7 +9293,7 @@ func TestReconcileNonBlockingBehavior(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(api.Scheme). WithObjects(hcluster, pullSecret, ingress). - WithStatusSubresource(hcluster). + WithStatusSubresource(hcluster, &capigcp.GCPCluster{}, &capiv1.Cluster{}). Build() // Apply test-specific mutations (fault injection) @@ -9164,6 +9316,9 @@ func TestReconcileNonBlockingBehavior(t *testing.T) { EnableOCPClusterMonitoring: true, CertRotationScale: 24 * time.Hour, HypershiftOperatorImage: "test-hso-image", + GCPOIDCStorageBucketName: "test-oidc-bucket", + GCSClient: newFakeGCSClient(), + HCPEgressBlockCIDRs: []string{"192.0.2.10/32"}, RegistryProvider: fakeReleaseProvider{ releaseProvider: mockReleaseProvider, metadataProvider: fakeMetadata, @@ -9175,7 +9330,7 @@ func TestReconcileNonBlockingBehavior(t *testing.T) { req := ctrl.Request{NamespacedName: crclient.ObjectKeyFromObject(hcluster)} log := zap.New(zap.UseDevMode(true), zap.Level(zapcore.InfoLevel)) - _, err := r.reconcile(t.Context(), req, log, hcluster) + _, err := r.Reconcile(ctrl.LoggerInto(t.Context(), log), req) if err == nil { t.Fatalf("expected reconcile to return an error containing %q, but got nil", tc.expectedErrSubstring) @@ -9185,6 +9340,9 @@ func TestReconcileNonBlockingBehavior(t *testing.T) { } tc.verifyResources(t, fakeClient, hcpNamespace, hcName) + if tc.verifyRecovery != nil { + tc.verifyRecovery(t, r, hcluster) + } }) } } diff --git a/support/conditions/conditions_test.go b/support/conditions/conditions_test.go index 3956670dcd49..433dba714363 100644 --- a/support/conditions/conditions_test.go +++ b/support/conditions/conditions_test.go @@ -109,6 +109,14 @@ func TestExpectedHCConditions(t *testing.T) { g.Expect(expected[hyperv1.GCPEndpointAvailable]).To(Equal(metav1.ConditionTrue)) }) } + t.Run("When GCP resource tags are configured, it should require valid configuration", func(t *testing.T) { + g := NewWithT(t) + hc := &hyperv1.HostedCluster{} + hc.Spec.Platform.Type = hyperv1.GCPPlatform + hc.Spec.Platform.GCP = &hyperv1.GCPPlatformSpec{ResourceTags: []hyperv1.GCPResourceTag{{Key: "environment", Value: "production"}}} + + g.Expect(ExpectedHCConditions(hc)[hyperv1.ValidHostedClusterConfiguration]).To(Equal(metav1.ConditionTrue)) + }) t.Run("When the control plane upgrades from 5.0 to 5.1, it should refresh credential expectations", func(t *testing.T) { g := NewWithT(t) hc := &hyperv1.HostedCluster{} diff --git a/support/globalconfig/infrastructure.go b/support/globalconfig/infrastructure.go index 5a95aeaaf3a6..c8dc9e2558ba 100644 --- a/support/globalconfig/infrastructure.go +++ b/support/globalconfig/infrastructure.go @@ -113,6 +113,7 @@ func ReconcileInfrastructure(infra *configv1.Infrastructure, hcp *hyperv1.Hosted IngressIPs: []string{}, } case hyperv1.GCPPlatform: + gcpStatusAlreadyExists := infra.Status.PlatformStatus.GCP != nil if infra.Status.PlatformStatus.GCP == nil { infra.Status.PlatformStatus.GCP = &configv1.GCPPlatformStatus{} } @@ -134,5 +135,23 @@ func ReconcileInfrastructure(infra *configv1.Infrastructure, hcp *hyperv1.Hosted }) } infra.Status.PlatformStatus.GCP.ResourceLabels = labels + var tags []configv1.GCPResourceTag + for _, tag := range hcp.Spec.Platform.GCP.ResourceTags { + parentID := tag.ParentID + if parentID == "" { + parentID = hcp.Spec.Platform.GCP.Project + } + tags = append(tags, configv1.GCPResourceTag{ + ParentID: parentID, + Key: tag.Key, + Value: tag.Value, + }) + } + // ResourceTags are immutable in the guest Infrastructure API. Do not + // backfill tags into an existing GCP status that was first written by an + // older control plane operator without them. + if !gcpStatusAlreadyExists || infra.Status.PlatformStatus.GCP.ResourceTags != nil || len(tags) == 0 { + infra.Status.PlatformStatus.GCP.ResourceTags = tags + } } } diff --git a/support/globalconfig/infrastructure_test.go b/support/globalconfig/infrastructure_test.go index a8bc3b484ed5..04159d75eb87 100644 --- a/support/globalconfig/infrastructure_test.go +++ b/support/globalconfig/infrastructure_test.go @@ -376,6 +376,76 @@ func TestReconcileInfrastructure(t *testing.T) { })) }, }, + { + name: "When GCP platform has resource tags, it should derive omitted parentIDs and preserve explicit parents", + inputInfra: InfrastructureConfig(), + inputHCP: func() *hyperv1.HostedControlPlane { + hcp := baseHCP(hyperv1.GCPPlatform) + hcp.Spec.Platform.GCP = &hyperv1.GCPPlatformSpec{ + Project: "customer-project", + Region: "us-central1", + ResourceTags: []hyperv1.GCPResourceTag{ + {Key: "Environment", Value: "production"}, + {ParentID: "customer-project", Key: "team", Value: "platform-123"}, + {ParentID: "123456789012", Key: "cost-center", Value: "shared"}, + {Key: "kubernetes-io", Value: "customer-defined"}, + }, + } + return hcp + }(), + verify: func(g Gomega, infra *configv1.Infrastructure) { + g.Expect(infra.Status.PlatformStatus.GCP.ResourceTags).To(Equal([]configv1.GCPResourceTag{ + {ParentID: "customer-project", Key: "Environment", Value: "production"}, + {ParentID: "customer-project", Key: "team", Value: "platform-123"}, + {ParentID: "123456789012", Key: "cost-center", Value: "shared"}, + {ParentID: "customer-project", Key: "kubernetes-io", Value: "customer-defined"}, + })) + }, + }, + { + name: "When GCP platform has no resource tags, it should leave guest tags omitted", + inputInfra: InfrastructureConfig(), + inputHCP: func() *hyperv1.HostedControlPlane { + hcp := baseHCP(hyperv1.GCPPlatform) + hcp.Spec.Platform.GCP = &hyperv1.GCPPlatformSpec{Project: "customer-project", ResourceTags: []hyperv1.GCPResourceTag{}} + return hcp + }(), + verify: func(g Gomega, infra *configv1.Infrastructure) { + g.Expect(infra.Status.PlatformStatus.GCP.ResourceTags).To(BeNil()) + }, + }, + { + name: "When GCP resource tags are omitted, it should leave guest tags omitted", + inputInfra: InfrastructureConfig(), + inputHCP: func() *hyperv1.HostedControlPlane { + hcp := baseHCP(hyperv1.GCPPlatform) + hcp.Spec.Platform.GCP = &hyperv1.GCPPlatformSpec{Project: "customer-project"} + return hcp + }(), + verify: func(g Gomega, infra *configv1.Infrastructure) { + g.Expect(infra.Status.PlatformStatus.GCP.ResourceTags).To(BeNil()) + }, + }, + { + name: "When existing GCP status has no resource tags, it should not backfill them", + inputInfra: func() *configv1.Infrastructure { + infra := InfrastructureConfig() + infra.Status.PlatformStatus = &configv1.PlatformStatus{GCP: &configv1.GCPPlatformStatus{ProjectID: "customer-project", Region: "us-central1"}} + return infra + }(), + inputHCP: func() *hyperv1.HostedControlPlane { + hcp := baseHCP(hyperv1.GCPPlatform) + hcp.Spec.Platform.GCP = &hyperv1.GCPPlatformSpec{ + Project: "customer-project", + Region: "us-central1", + ResourceTags: []hyperv1.GCPResourceTag{{Key: "environment", Value: "production"}}, + } + return hcp + }(), + verify: func(g Gomega, infra *configv1.Infrastructure) { + g.Expect(infra.Status.PlatformStatus.GCP.ResourceTags).To(BeNil()) + }, + }, { name: "When GCP PlatformStatus is already initialized, it should update existing fields from HCP spec", inputInfra: func() *configv1.Infrastructure { diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/gcp.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/gcp.go index b298c7acdf91..f415f8c08656 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/gcp.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/gcp.go @@ -47,6 +47,50 @@ type GCPResourceLabel struct { Value *string `json:"value,omitempty"` } +// GCPResourceTag selects a pre-existing Google Cloud Resource Manager tag by +// its parent, short key, and short value. The tag must be defined in the +// HostedCluster project or an organization before the cluster is created. +// See https://cloud.google.com/resource-manager/docs/tags/tags-overview. +type GCPResourceTag struct { + // parentID is the ID of the project or organization that defines the TagKey. + // When omitted, the HostedCluster's GCP project is used. + // An explicit value must be the HostedCluster's GCP project ID or a numeric + // organization ID. A numeric value is treated as an organization ID; project + // numbers are not supported. + // An organization ID consists of 1 to 32 decimal digits and must not start with 0. + // A project ID is 6 to 30 characters long, contains only lowercase letters, digits + // and hyphens, starts with a letter and must not end with a hyphen. + // + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=32 + // +kubebuilder:validation:XValidation:rule="self.matches('(^[1-9][0-9]{0,31}$)|(^[a-z][a-z0-9-]{4,28}[a-z0-9]$)')",message="parentID must be a numeric organization ID or a valid GCP project ID" + ParentID string `json:"parentID,omitempty"` + + // key is required and is the short name of the pre-existing Resource Manager TagKey. + // A key is 1-63 characters, begins and ends with + // an ASCII alphanumeric character, and may contain letters, digits, '.', + // '_', or '-' between them. + // + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:XValidation:rule="self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.-]{0,61}[a-zA-Z0-9])?$')",message="key must start and end with an ASCII letter or digit and contain only ASCII letters, digits, '.', '_' or '-'" + Key string `json:"key,omitempty"` + + // value is required and is the short name of the pre-existing Resource Manager TagValue for + // key. Exactly one value for a TagKey can be attached to a resource. A value + // is 1-63 characters, begins and ends with an ASCII alphanumeric character, + // and may contain ASCII letters and digits, `_-.@%=+:,*#&()[]{}`, and + // spaces between them. + // + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:XValidation:rule="self.matches('^[a-zA-Z0-9]([0-9A-Za-z_.@%=+:,*#&()\\\\[\\\\]{}\\\\- ]{0,61}[a-zA-Z0-9])?$')",message="value must start and end with an ASCII letter or digit and contain only ASCII letters, digits, spaces or the characters _-.@%=+:,*#&()[]{}" + Value string `json:"value,omitempty"` +} + // GCPEndpointAccessType defines the endpoint access type for GCP clusters. // Equivalent to AWS EndpointAccessType but adapted for GCP networking model. type GCPEndpointAccessType string @@ -107,6 +151,8 @@ type GCPNetworkConfig struct { // +kubebuilder:validation:XValidation:rule="self.workloadIdentity.serviceAccountsEmails.storage.contains('@') && self.workloadIdentity.serviceAccountsEmails.storage.endsWith('@' + self.project + '.iam.gserviceaccount.com')",message="storage service account must belong to the same project" // +kubebuilder:validation:XValidation:rule="self.workloadIdentity.serviceAccountsEmails.imageRegistry.contains('@') && self.workloadIdentity.serviceAccountsEmails.imageRegistry.endsWith('@' + self.project + '.iam.gserviceaccount.com')",message="imageRegistry service account must belong to the same project" // +kubebuilder:validation:XValidation:rule="self.workloadIdentity.serviceAccountsEmails.network.endsWith('@' + self.project + '.iam.gserviceaccount.com')",message="network service account must belong to the same project" +// +kubebuilder:validation:XValidation:rule="has(self.resourceTags) == has(oldSelf.resourceTags)",message="resourceTags may only be configured during installation",fieldPath=".resourceTags" +// +kubebuilder:validation:XValidation:rule="!has(self.resourceTags) || self.resourceTags.all(t, !has(t.parentID) || t.parentID.matches('^[1-9][0-9]{0,31}$') || t.parentID == self.project)",message="resourceTags parentID must be an organization ID or the GCP platform project",fieldPath=".resourceTags" type GCPPlatformSpec struct { // project is the GCP project ID. // A valid project ID must satisfy the following rules: @@ -161,6 +207,45 @@ type GCPPlatformSpec struct { // +kubebuilder:validation:MaxItems=60 ResourceLabels []GCPResourceLabel `json:"resourceLabels,omitempty"` + // resourceTags are pre-existing Google Cloud Resource Manager tags to apply to + // GCP resources created for the hosted cluster. + // Currently, tags are applied to newly provisioned persistent disks and, when the + // image registry is enabled, to its storage bucket. Other GCP resources, such as + // worker VMs and their boot disks, are not tagged. + // The set of tagged resources is subject to change over time. + // + // When omitted, no tags are applied through this setting. When specified, + // the list must contain 1 to 50 entries, and each short key must be unique + // across the list, even when the tags have different parents. A tag's + // parentID defaults to the HostedCluster's GCP project if omitted. + // Google Cloud permits at most 50 directly attached tags per resource, + // including tags attached by other actors. Coordinate with external tagging + // automation to leave sufficient capacity; admission does not reserve tag slots. + // + // The referenced tags must already exist, and the storage and imageRegistry + // service accounts must have permission to use them. Neither is checked when the + // HostedCluster is created. When a tag cannot be applied, provisioning of the + // affected disk or bucket can fail, and the error is reported in the persistent + // volume claim's events or the image registry operator's status. + // + // resourceTags can only be set when the HostedCluster is created. It cannot be + // added, removed or changed afterwards. If the configured list must change, + // the HostedCluster must be recreated. + // Resource tags require a control plane operator image that supports GCP resource + // tag propagation. When that capability is unavailable, reconciliation fails + // with a message identifying resourceTags on the ReconciliationSucceeded + // condition, and dependent component deployments are blocked until a + // supporting control plane operator image is configured. + // + // +optional + // +listType=atomic + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=50 + // +immutable + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="resourceTags are immutable" + // +kubebuilder:validation:XValidation:rule="self.all(t, self.exists_one(u, u.key == t.key))",message="resourceTags keys must be unique across all parents" + ResourceTags []GCPResourceTag `json:"resourceTags,omitempty"` + // workloadIdentity configures Workload Identity Federation for the cluster. // This enables secure, short-lived token-based authentication without storing // long-term service account keys. These fields are immutable after cluster creation @@ -349,6 +434,7 @@ type GCPServiceAccountsEmails struct { // that manages GCS storage for the internal container image registry. // This GSA requires the following IAM roles: // - roles/storage.admin (Storage Admin - for creating and managing GCS buckets and objects) + // - roles/resourcemanager.tagUser (Tag User - for applying resource tags to the bucket) // See cmd/infra/gcp/iam-bindings.json for the authoritative role definitions. // Format: service-account-name@project-id.iam.gserviceaccount.com // diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go index ff15d4f1719d..97cffb545fd2 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go @@ -1486,6 +1486,8 @@ func PlatformTypes() []PlatformType { // PlatformSpec specifies the underlying infrastructure provider for the cluster // and is used to configure platform specific behavior. +// +// +openshift:validation:FeatureGateAwareXValidation:featureGate=GCPPlatform,rule="(has(self.gcp) && has(self.gcp.resourceTags)) == (has(oldSelf.gcp) && has(oldSelf.gcp.resourceTags))",message="resourceTags may only be configured during installation" type PlatformSpec struct { // type is the type of infrastructure provider for the cluster. // diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go index 4c0459433bcf..6b2522e73d84 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -2146,6 +2146,11 @@ func (in *GCPPlatformSpec) DeepCopyInto(out *GCPPlatformSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.ResourceTags != nil { + in, out := &in.ResourceTags, &out.ResourceTags + *out = make([]GCPResourceTag, len(*in)) + copy(*out, *in) + } out.WorkloadIdentity = in.WorkloadIdentity } @@ -2302,6 +2307,21 @@ func (in *GCPResourceReference) DeepCopy() *GCPResourceReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GCPResourceTag) DeepCopyInto(out *GCPResourceTag) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GCPResourceTag. +func (in *GCPResourceTag) DeepCopy() *GCPResourceTag { + if in == nil { + return nil + } + out := new(GCPResourceTag) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GCPServiceAccountsEmails) DeepCopyInto(out *GCPServiceAccountsEmails) { *out = *in