CNTRLPLANE-4811: trigger rollout on trust bundle ConfigMap content changes

Containerfile.control-plane

@@ -22,6 +22,7 @@ LABEL io.openshift.hypershift.control-plane-operator-manages-ignition-server=tru
 LABEL io.openshift.hypershift.control-plane-operator-manages.cluster-machine-approver=true
 LABEL io.openshift.hypershift.control-plane-operator-manages.cluster-autoscaler=true
 LABEL io.openshift.hypershift.control-plane-operator-manages.decompress-decode-config=true
+LABEL io.openshift.hypershift.control-plane-operator-manages.rendered-trust-bundle-validation=true
 LABEL io.openshift.hypershift.control-plane-operator-creates-aws-sg=true
 LABEL io.openshift.hypershift.control-plane-operator-applies-management-kas-network-policy-label=true
 LABEL io.openshift.hypershift.restricted-psa=true

Dockerfile

@@ -28,6 +28,7 @@ LABEL io.openshift.hypershift.control-plane-operator-manages-ignition-server=tru
 LABEL io.openshift.hypershift.control-plane-operator-manages.cluster-machine-approver=true
 LABEL io.openshift.hypershift.control-plane-operator-manages.cluster-autoscaler=true
 LABEL io.openshift.hypershift.control-plane-operator-manages.decompress-decode-config=true
+LABEL io.openshift.hypershift.control-plane-operator-manages.rendered-trust-bundle-validation=true
 LABEL io.openshift.hypershift.control-plane-operator-creates-aws-sg=true
 LABEL io.openshift.hypershift.control-plane-operator-applies-management-kas-network-policy-label=true
 LABEL io.openshift.hypershift.restricted-psa=true

Dockerfile.control-plane

@@ -22,6 +22,7 @@ LABEL io.openshift.hypershift.control-plane-operator-manages-ignition-server=tru
 LABEL io.openshift.hypershift.control-plane-operator-manages.cluster-machine-approver=true
 LABEL io.openshift.hypershift.control-plane-operator-manages.cluster-autoscaler=true
 LABEL io.openshift.hypershift.control-plane-operator-manages.decompress-decode-config=true
+LABEL io.openshift.hypershift.control-plane-operator-manages.rendered-trust-bundle-validation=true
 LABEL io.openshift.hypershift.control-plane-operator-creates-aws-sg=true
 LABEL io.openshift.hypershift.control-plane-operator-applies-management-kas-network-policy-label=true
 LABEL io.openshift.hypershift.restricted-psa=true

Dockerfile.dev

@@ -35,6 +35,7 @@ LABEL io.openshift.hypershift.control-plane-operator-manages-ignition-server=tru
 LABEL io.openshift.hypershift.control-plane-operator-manages.cluster-machine-approver=true
 LABEL io.openshift.hypershift.control-plane-operator-manages.cluster-autoscaler=true
 LABEL io.openshift.hypershift.control-plane-operator-manages.decompress-decode-config=true
+LABEL io.openshift.hypershift.control-plane-operator-manages.rendered-trust-bundle-validation=true
 LABEL io.openshift.hypershift.control-plane-operator-creates-aws-sg=true
 LABEL io.openshift.hypershift.control-plane-operator-applies-management-kas-network-policy-label=true
 LABEL io.openshift.hypershift.restricted-psa=true

docs/content/reference/nodepool-rollouts.md

@@ -34,7 +34,38 @@ Changes to the following fields alter the configuration hash that the controller
 
 - **`HostedCluster.spec.pullSecret`** — a change in the **name** of the referenced Secret triggers a rollout. Changing the content of the Secret without changing the name does not trigger a rollout.
 
-- **`HostedCluster.spec.additionalTrustBundle`** — same behavior as `pullSecret`: only a change in the referenced ConfigMap **name** triggers a rollout.
+- **`HostedCluster.spec.additionalTrustBundle`** — changing the `ca-bundle.crt` content triggers a rollout across all NodePools. Switching to another ConfigMap with identical content does not itself trigger a rollout.
+
+- **`HostedCluster.spec.configuration.proxy.trustedCA`** — changing the referenced ConfigMap **name** or the `ca-bundle.crt` content triggers a rollout across all NodePools.
+
+Trust-bundle reconciliation requires a CPO/ignition-server image advertising
+`io.openshift.hypershift.control-plane-operator-manages.rendered-trust-bundle-validation`.
+The ignition server checks the hash of the actual combined additional and proxy CA
+bundle rendered for the MCO before caching a payload. A delayed ConfigMap update
+therefore delays payload generation instead of installing stale certificates.
+Older CPO images need this validation backported (and the capability label added);
+upgrading only the HyperShift Operator is insufficient. With an unsupported CPO,
+trust-bundle reconciliation stops before bootstrap changes or old-token cleanup.
+
+During formula-only migration, Karpenter retains its existing bootstrap/drift
+identity and persists a separate versioned content baseline on its durable
+NodeClass resource. Later reconciles compare content against that baseline,
+including after a controller restart. A subsequent CA change produces a new
+identity and token. Regular NodePools persist their baseline on the NodePool;
+the migration comparison is independent of the infrastructure platform.
+
+!!! note "HyperShift Operator upgrade and trust-bundle hashing"
+
+    When upgrading to a HyperShift Operator that hashes trust-bundle ConfigMap **content**
+    (instead of only the ConfigMap name), NodePools that already reference a trust bundle do
+    **not** roll solely because of that formula change. On the first reconcile after upgrade,
+    the controller seeds the versioned rollout baseline
+    (`nodePoolCurrentRolloutConfigV2`) and records `nodePoolConfigHashVersion=v2`.
+    It retains the completed payload hash and the bootstrap Secret references.
+    CA edits made before this first reconcile are absorbed into the baseline.
+    Subsequent CA-content or other rollout-relevant spec changes trigger a rollout;
+    management-only image changes do not. In-progress rollouts retain their deployed
+    target, while a newer release request can still supersede it.
 
 - **`HostedCluster.spec.imageContentSources`** — changes to image content source policies managed at the HostedCluster level produce an additional core ignition config that alters the configuration hash.
 
@@ -171,7 +202,8 @@ These conditions are set to `True` while the corresponding rollout is in progres
 | `NodePool.spec.config` | Yes | The changed NodePool |
 | `NodePool.spec.tuningConfig` | Yes | The changed NodePool |
 | `HostedCluster.spec.pullSecret` (name change) | Yes | All NodePools |
-| `HostedCluster.spec.additionalTrustBundle` (name change) | Yes | All NodePools |
+| `HostedCluster.spec.additionalTrustBundle` (content change) | Yes | All NodePools |
+| `HostedCluster.spec.configuration.proxy.trustedCA` (name or content change) | Yes | All NodePools |
 | `HostedCluster.spec.imageContentSources` | Yes | All NodePools |
 | `HostedCluster.spec.configuration.proxy` | Yes | All NodePools |
 | `HostedCluster.spec.configuration.image` | Yes | All NodePools |

hack/tools/hypershiftlinter/analyzers/e2eutilallowlist/allowlist.go

@@ -722,26 +722,6 @@ var allowlist = map[string]map[string][]string{
 			"WaitForReadyNodesByNodePool",
 			"WithTimeout",
 		},
-		"NodePoolTrustBundleTest/It(\"should propagate and remove additional trust bundle to/from the hosted cluster\")#1": {
-			"Condition",
-			"Condition.Status",
-			"Condition.Type",
-			"ConditionPredicate",
-			"EventuallyNotFound",
-			"EventuallyObject",
-			"NameGenerator.GenerateName",
-			"Predicate",
-			"SimpleNameGenerator",
-			"UpdateObject",
-			"Version418",
-			"Version422",
-			"WaitForReadyNodesByNodePool",
-			"WithInterval",
-			"WithTimeout",
-		},
-		"NodePoolTrustBundleTest/It(\"should propagate and remove additional trust bundle to/from the hosted cluster\")#1/DeferCleanup()#3": {
-			"UpdateObject",
-		},
 		"buildMachineConfigVerificationDaemonSet": {
 			"NameGenerator.GenerateName",
 			"SimpleNameGenerator",

hypershift-operator/controllers/nodepool/annotation_seeding_test.go

@@ -14,6 +14,7 @@ import (
 )
 
 func TestSeedRolloutAnnotation(t *testing.T) {
+	t.Run("When trust bundle hashing changes, it should preserve migration and subsequent rollout behavior", testSeedRolloutAnnotationTrustBundleMigration)
 	config := &rolloutConfig{
 		releaseImage: &releaseinfo.ReleaseImage{ImageStream: &imagev1.ImageStream{ObjectMeta: metav1.ObjectMeta{Name: "4.18.5"}}},
 	}
@@ -96,6 +97,7 @@ func TestSeedRolloutAnnotation(t *testing.T) {
 				deployedBootstrapHash: tc.deployedHash,
 			}
 			seedRolloutAnnotation(token)
+			tc.expected[nodePoolAnnotationConfigHashVersion] = CurrentConfigHashVersion
 			NewWithT(t).Expect(nodePool.Annotations).To(Equal(tc.expected))
 		})
 	}

hypershift-operator/controllers/nodepool/capi_test.go

@@ -3005,7 +3005,7 @@ func TestPropagateVersionAndTemplate(t *testing.T) {
 		rc := &rolloutConfig{
 			rolloutMcoRawConfig:       "user-config",
 			pullSecretName:            "pull",
-			additionalTrustBundleName: "trust",
+			additionalTrustBundleHash: "trust",
 			rolloutGlobalConfig:       "global",
 			mcoRawConfig:              "old-haproxy\n---\nuser-config",
 			haproxyRawConfig:          "old-haproxy",
@@ -3241,7 +3241,7 @@ func TestPropagateVersionAndTemplateToMachineSet(t *testing.T) {
 		rc := &rolloutConfig{
 			rolloutMcoRawConfig:       "user-config",
 			pullSecretName:            "pull",
-			additionalTrustBundleName: "trust",
+			additionalTrustBundleHash: "trust",
 			rolloutGlobalConfig:       "global",
 			mcoRawConfig:              "old-haproxy\n---\nuser-config",
 			haproxyRawConfig:          "old-haproxy",
@@ -3869,7 +3869,7 @@ func TestReconcileMachineSetStatus(t *testing.T) {
 		rc := &rolloutConfig{
 			rolloutMcoRawConfig:       "user-config",
 			pullSecretName:            "pull",
-			additionalTrustBundleName: "trust",
+			additionalTrustBundleHash: "trust",
 			rolloutGlobalConfig:       "global",
 			mcoRawConfig:              "old-haproxy\n---\nuser-config",
 			haproxyRawConfig:          "old-haproxy",

hypershift-operator/controllers/nodepool/conditions.go

@@ -2,6 +2,7 @@ package nodepool
 
 import (
 	"context"
+	coreerrors "errors"
 	"fmt"
 	"net/netip"
 	"strconv"
@@ -428,14 +429,13 @@ func (r *NodePoolReconciler) validMachineConfigCondition(ctx context.Context, no
 	}
 	_, err = NewConfigGenerator(ctx, r.Client, hcluster, nodePool, releaseImage, haproxyRawConfig, controlPlaneNamespace, resolvedRHELStream)
 	if err != nil {
-		SetStatusCondition(&nodePool.Status.Conditions, hyperv1.NodePoolCondition{
-			Type:               hyperv1.NodePoolValidMachineConfigConditionType,
-			Status:             corev1.ConditionFalse,
-			Reason:             hyperv1.NodePoolValidationFailedReason,
-			Message:            err.Error(),
-			ObservedGeneration: nodePool.Generation,
-		})
-		return &ctrl.Result{}, fmt.Errorf("failed to generate config: %w", err)
+		result, condErr := setValidMachineConfigConditionForError(nodePool, err)
+		if condErr == nil {
+			// Trust-bundle ConfigMap problems soft-fail; ValidMachineConfig=False already
+			// surfaces the problem and ConfigMap watches requeue when restored.
+			log.Info("HostedCluster trust bundle ConfigMap is missing or invalid", "error", err.Error())
+		}
+		return result, condErr
 	}
 
 	SetStatusCondition(&nodePool.Status.Conditions, hyperv1.NodePoolCondition{
@@ -448,6 +448,32 @@ func (r *NodePoolReconciler) validMachineConfigCondition(ctx context.Context, no
 	return nil, nil
 }
 
+// setValidMachineConfigConditionForError sets ValidMachineConfig=False for a config generation error.
+// Missing or malformed HostedCluster trust-bundle ConfigMaps soft-fail (nil error) so transient
+// delete/recreate windows do not hard-fail NodePool reconciliation. Other config errors still
+// return an error for exponential backoff.
+func setValidMachineConfigConditionForError(nodePool *hyperv1.NodePool, err error) (*ctrl.Result, error) {
+	reason := hyperv1.NodePoolValidationFailedReason
+	var trustBundleErr *TrustBundleConfigError
+	softFail := coreerrors.As(err, &trustBundleErr)
+	if softFail && apierrors.IsNotFound(err) {
+		reason = hyperv1.NodePoolNotFoundReason
+	}
+
+	SetStatusCondition(&nodePool.Status.Conditions, hyperv1.NodePoolCondition{
+		Type:               hyperv1.NodePoolValidMachineConfigConditionType,
+		Status:             corev1.ConditionFalse,
+		Reason:             reason,
+		Message:            err.Error(),
+		ObservedGeneration: nodePool.Generation,
+	})
+	if softFail {
+		// Reconciling won't solve the input problem; a ConfigMap create/update will requeue.
+		return &ctrl.Result{}, nil
+	}
+	return &ctrl.Result{}, fmt.Errorf("failed to generate config: %w", err)
+}
+
 func (r *NodePoolReconciler) updatingConfigCondition(ctx context.Context, nodePool *hyperv1.NodePool, hcluster *hyperv1.HostedCluster) (*ctrl.Result, error) {
 	log := ctrl.LoggerFrom(ctx)
 	token, err := r.token(ctx, hcluster, nodePool)

hypershift-operator/controllers/nodepool/conditions_test.go

@@ -19,7 +19,9 @@ import (
 	"github.com/openshift/api/image/docker10"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/utils/ptr"
 
 	"sigs.k8s.io/cluster-api/api/core/v1beta2"
@@ -29,6 +31,60 @@ import (
 	"github.com/blang/semver"
 )
 
+func TestSetValidMachineConfigConditionForError(t *testing.T) {
+	testCases := []struct {
+		name           string
+		err            error
+		expectedReason string
+		expectError    bool
+	}{
+		{
+			name: "When trust bundle ConfigMap is not found, it should soft-fail with NotFound reason",
+			err: &TrustBundleConfigError{err: fmt.Errorf("cannot get ConfigMap clusters/user-ca: %w",
+				apierrors.NewNotFound(schema.GroupResource{Resource: "configmaps"}, "user-ca"))},
+			expectedReason: hyperv1.NodePoolNotFoundReason,
+			expectError:    false,
+		},
+		{
+			name: "When trust bundle ConfigMap is missing ca-bundle.crt, it should soft-fail with ValidationFailed reason",
+			err: &TrustBundleConfigError{err: fmt.Errorf("ConfigMap clusters/user-ca missing %q key",
+				"ca-bundle.crt")},
+			expectedReason: hyperv1.NodePoolValidationFailedReason,
+			expectError:    false,
+		},
+		{
+			name:           "When a non-trust-bundle config error occurs, it should hard-fail with ValidationFailed reason",
+			err:            fmt.Errorf("expected 3 core ignition configs, found 0"),
+			expectedReason: hyperv1.NodePoolValidationFailedReason,
+			expectError:    true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+			nodePool := &hyperv1.NodePool{
+				ObjectMeta: metav1.ObjectMeta{Name: "workers", Namespace: "clusters", Generation: 2},
+			}
+
+			result, err := setValidMachineConfigConditionForError(nodePool, tc.err)
+			g.Expect(result).NotTo(BeNil())
+			if tc.expectError {
+				g.Expect(err).To(HaveOccurred())
+			} else {
+				g.Expect(err).NotTo(HaveOccurred())
+			}
+
+			condition := FindStatusCondition(nodePool.Status.Conditions, hyperv1.NodePoolValidMachineConfigConditionType)
+			g.Expect(condition).NotTo(BeNil())
+			g.Expect(condition.Status).To(Equal(corev1.ConditionFalse))
+			g.Expect(condition.Reason).To(Equal(tc.expectedReason))
+			g.Expect(condition.Message).To(Equal(tc.err.Error()))
+			g.Expect(condition.ObservedGeneration).To(Equal(nodePool.Generation))
+		})
+	}
+}
+
 func TestIgnitionEndpointAvailableCondition(t *testing.T) {
 	const ignitionHostname = "ignition-example.service.hypershift.example.com"