feat(controller): make reconciles restart-safe

charts/solo-operator/templates/deployment.yaml

@@ -29,7 +29,7 @@ spec:
       serviceAccountName: {{ include "solo-operator.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: {{ add .Values.controller.gracefulShutdownTimeoutSeconds 15 }}
       containers:
         - name: manager
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
@@ -58,6 +58,7 @@ spec:
             {{- with .Values.controller.nodeActiveAlertInterval }}
             - --node-active-alert-interval={{ . }}
             {{- end }}
+            - --graceful-shutdown-timeout={{ .Values.controller.gracefulShutdownTimeoutSeconds }}s
             {{- $allowedNamespaces := include "solo-operator.allowedNamespaces" . }}
             {{- if ne $allowedNamespaces "_all_namespaces_placeholder" }}
             - --allowed-namespaces={{ $allowedNamespaces | replace " " "," }}

charts/solo-operator/tests/shutdown_test.yaml

@@ -0,0 +1,23 @@
+suite: deployment shutdown window
+templates:
+  - templates/deployment.yaml
+tests:
+  - it: defaults to a 30s drain inside a 45s grace period
+    asserts:
+      - equal:
+          path: spec.template.spec.terminationGracePeriodSeconds
+          value: 45
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: --graceful-shutdown-timeout=30s
+
+  - it: derives the grace period from the drain timeout
+    set:
+      controller.gracefulShutdownTimeoutSeconds: 60
+    asserts:
+      - equal:
+          path: spec.template.spec.terminationGracePeriodSeconds
+          value: 75
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: --graceful-shutdown-timeout=60s

charts/solo-operator/values.yaml

@@ -152,3 +152,5 @@ controller:
   # initial deadline fires. Shorter values give the node operator more frequent
   # visibility. Empty uses the operator default (5m).
   nodeActiveAlertInterval: 5m
+  # Best-effort SIGTERM drain in seconds; the pod grace period is this plus 15s.
+  gracefulShutdownTimeoutSeconds: 30

cmd/main.go

@@ -71,6 +71,7 @@ func main() {
 	var allowedNamespaces string
 	var nodeActiveTimeout time.Duration
 	var nodeActiveAlertInterval time.Duration
+	var gracefulShutdownTimeout time.Duration
 	var showVersion bool
 	var logLevel string
 	var consoleLogging bool
@@ -106,6 +107,9 @@ func main() {
 	flag.DurationVar(&nodeActiveAlertInterval, "node-active-alert-interval", 5*time.Minute,
 		"Interval between repeated NodeActiveDeadlineExceeded warning events after the initial "+
 			"deadline fires. Shorter values give the node operator more frequent visibility.")
+	flag.DurationVar(&gracefulShutdownTimeout, "graceful-shutdown-timeout", 30*time.Second,
+		"How long in-flight reconciles may run after SIGTERM before the manager stops. "+
+			"Must be shorter than the pod's terminationGracePeriodSeconds.")
 
 	flag.Parse()
 
@@ -185,6 +189,9 @@ func main() {
 		Metrics:                metricsServerOptions,
 		HealthProbeBindAddress: probeAddr,
 		LeaderElection:         enableLeaderElection,
+		// Safe because main exits as soon as mgr.Start returns.
+		LeaderElectionReleaseOnCancel: true,
+		GracefulShutdownTimeout:       &gracefulShutdownTimeout,
 		// LeaderElectionID matches the API group (operator.solo.hiero.org) so the lock
 		// name is consistent with CRD group, finalizer, and label key naming conventions.
 		LeaderElectionID: "operator.solo.hiero.org",

config/manager/manager.yaml

@@ -61,4 +61,4 @@ spec:
       - name: tmp
         emptyDir: {}
       serviceAccountName: controller-manager
-      terminationGracePeriodSeconds: 10
+      terminationGracePeriodSeconds: 45

internal/controller/networkgenesis_controller.go

@@ -9,6 +9,7 @@ import (
 	"github.com/automa-saga/errx"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/events"
@@ -210,6 +211,7 @@ func (r *NetworkGenesisReconciler) succeed(ctx context.Context, ng *operatorv1al
 		genesis.ClusterConfigMapName,
 	}
 	ng.Status.GeneratedConfigMaps = generatedCMs
+	apimeta.RemoveStatusCondition(&ng.Status.Conditions, networkGenesisConditionFailed)
 	setNetworkGenesisCondition(&ng.Status.Conditions, metav1.Condition{
 		Type:               networkGenesisConditionReady,
 		Status:             metav1.ConditionTrue,
@@ -226,6 +228,7 @@ func (r *NetworkGenesisReconciler) fail(ctx context.Context, ng *operatorv1alpha
 	ng.Status.Phase = operatorv1alpha1.PhaseFailed
 	ng.Status.ObservedGeneration = ng.Generation
 	ng.Status.CompletionTime = &now
+	apimeta.RemoveStatusCondition(&ng.Status.Conditions, networkGenesisConditionReady)
 	setNetworkGenesisCondition(&ng.Status.Conditions, metav1.Condition{
 		Type:               networkGenesisConditionFailed,
 		Status:             metav1.ConditionTrue,
@@ -241,6 +244,7 @@ func (r *NetworkGenesisReconciler) rejectMutation(ctx context.Context, ng *opera
 	now := metav1.Now()
 	ng.Status.Phase = operatorv1alpha1.PhaseFailed
 	ng.Status.ObservedGeneration = ng.Generation
+	apimeta.RemoveStatusCondition(&ng.Status.Conditions, networkGenesisConditionReady)
 	setNetworkGenesisCondition(&ng.Status.Conditions, metav1.Condition{
 		Type:               networkGenesisConditionFailed,
 		Status:             metav1.ConditionTrue,

internal/controller/networkupgradeexecute_controller.go

@@ -14,6 +14,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -208,8 +209,10 @@ func (r *NetworkUpgradeExecuteReconciler) Reconcile(ctx context.Context, req ctr
 			return ctrl.Result{}, err
 		}
 
-		now := metav1.Now()
-		execute.Status.StartTime = &now
+		if execute.Status.StartTime == nil {
+			now := metav1.Now()
+			execute.Status.StartTime = &now
+		}
 		execute.Status.ObservedGeneration = execute.Generation
 
 		// Both modes transition to ReadyForProvisionerDaemon; the difference is who
@@ -403,72 +406,10 @@ func (r *NetworkUpgradeExecuteReconciler) reconcilePendingNodeUpgrade(ctx contex
 			}
 		}
 
-		prepare, err := findPrepare(ctx, r.Client, execute.Namespace, execute.Spec.OperationID)
-		if err != nil {
-			return ctrl.Result{}, err
-		}
-
-		if _, prefetchOK := prefetchCompleted(prepare); prefetchOK {
-			// Prefetch succeeded — skip registry re-validation. The images are
-			// likely cached on nodes; the kubelet uses them via IfNotPresent.
-			common.Info(ctx, operatorv1alpha1.ReasonPrefetchCompleted).Msg("NetworkUpgradeExecute: images prefetched, skipping registry re-validation")
-			ss := execute.Spec.SelectedSources
-			execute.Status.SelectedSources = &ss
-			setCondition(&execute.Status.Conditions, metav1.Condition{
-				Type:    operatorv1alpha1.CondRegistrySelected,
-				Status:  metav1.ConditionTrue,
-				Reason:  operatorv1alpha1.ReasonPrefetchCompleted.String(),
-				Message: "images prefetched on nodes; registry re-validation skipped",
-			})
-		} else {
-			rng := rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64()))
-			checker, err := capsuleRegistryChecker(ctx, r.Client, r.Checker, execute.Namespace, execute.Spec.ConsensusCapsule, r.AllowedRegistryHosts)
-			if err != nil {
-				return ctrl.Result{}, err
-			}
-			skipUC := map[operatorv1alpha1.ComponentName]bool{operatorv1alpha1.ComponentUC: true}
-			updated, registryOK := registry.RevalidateComponents(
-				ctx, checker,
-				execute.Spec.Images, execute.Spec.SelectedSources,
-				checkTimeout, rng, r.StrategyName,
-				skipUC,
-				func(name operatorv1alpha1.ComponentName, res registry.FailoverResult) {
-					r.eventf(ctx, execute, corev1.EventTypeWarning, operatorv1alpha1.EventRegistryFailover,
-						"%s: registry %s unreachable, failed over to %s (digest: %s)",
-						name, res.FailedImage, res.Selected.Image, res.Selected.Digest)
-				},
-				func(name operatorv1alpha1.ComponentName, err error) {
-					r.eventf(ctx, execute, corev1.EventTypeWarning, operatorv1alpha1.EventRegistryInvalid,
-						"%s: %v", name, err)
-				},
-			)
-			if !registryOK {
-				now := metav1.Now()
-				execute.Status.Phase = operatorv1alpha1.ExecutePhaseFailed
-				execute.Status.CompletionTime = &now
-				execute.Status.ObservedGeneration = execute.Generation
-				setCondition(&execute.Status.Conditions, metav1.Condition{
-					Type:    operatorv1alpha1.CondRegistrySelected,
-					Status:  metav1.ConditionFalse,
-					Reason:  operatorv1alpha1.ReasonRegistryInvalid.String(),
-					Message: "no valid registry available for one or more components",
-				})
-				if err := r.Status().Update(ctx, execute); err != nil {
-					return ctrl.Result{}, wrapStatusUpdate(err, "updating status to failed")
-				}
-				r.eventf(ctx, execute, corev1.EventTypeWarning, operatorv1alpha1.EventUpgradeFailed,
-					"Registry validation failed for operationId %q", execute.Spec.OperationID)
-				return ctrl.Result{}, errx.WithHints(
-					errx.WithReason(errReconcile.New("no valid registry for one or more components"), ReasonNoValidRegistry),
-					"Confirm at least one configured registry serves a valid image for each component")
+		if !apimeta.IsStatusConditionTrue(execute.Status.Conditions, operatorv1alpha1.CondRegistrySelected) || execute.Status.SelectedSources == nil {
+			if res, err := r.selectSources(ctx, execute); res != nil {
+				return *res, err
 			}
-			execute.Status.SelectedSources = updated
-			setCondition(&execute.Status.Conditions, metav1.Condition{
-				Type:    operatorv1alpha1.CondRegistrySelected,
-				Status:  metav1.ConditionTrue,
-				Reason:  operatorv1alpha1.ReasonSelectionComplete.String(),
-				Message: "all selected registries healthy",
-			})
 		}
 
 		patch := client.MergeFrom(capsule.DeepCopy())
@@ -583,6 +524,86 @@ func (r *NetworkUpgradeExecuteReconciler) reconcilePendingNodeUpgrade(ctx contex
 	}
 }
 
+// selectSources persists the registry choice before the capsule patch so a restart reuses it.
+func (r *NetworkUpgradeExecuteReconciler) selectSources(ctx context.Context, execute *operatorv1alpha1.NetworkUpgradeExecute) (*ctrl.Result, error) {
+	prepare, err := findPrepare(ctx, r.Client, execute.Namespace, execute.Spec.OperationID)
+	if err != nil {
+		return &ctrl.Result{}, err
+	}
+
+	if _, prefetchOK := prefetchCompleted(prepare); prefetchOK {
+		// Prefetch succeeded — skip registry re-validation. The images are
+		// likely cached on nodes; the kubelet uses them via IfNotPresent.
+		common.Info(ctx, operatorv1alpha1.ReasonPrefetchCompleted).Msg("NetworkUpgradeExecute: images prefetched, skipping registry re-validation")
+		ss := execute.Spec.SelectedSources
+		execute.Status.SelectedSources = &ss
+		setCondition(&execute.Status.Conditions, metav1.Condition{
+			Type:    operatorv1alpha1.CondRegistrySelected,
+			Status:  metav1.ConditionTrue,
+			Reason:  operatorv1alpha1.ReasonPrefetchCompleted.String(),
+			Message: "images prefetched on nodes; registry re-validation skipped",
+		})
+	} else {
+		rng := rand.New(rand.NewPCG(rand.Uint64(), rand.Uint64()))
+		checker, err := capsuleRegistryChecker(ctx, r.Client, r.Checker, execute.Namespace, execute.Spec.ConsensusCapsule, r.AllowedRegistryHosts)
+		if err != nil {
+			return &ctrl.Result{}, err
+		}
+		skipUC := map[operatorv1alpha1.ComponentName]bool{operatorv1alpha1.ComponentUC: true}
+		updated, registryOK := registry.RevalidateComponents(
+			ctx, checker,
+			execute.Spec.Images, execute.Spec.SelectedSources,
+			checkTimeout, rng, r.StrategyName,
+			skipUC,
+			func(name operatorv1alpha1.ComponentName, res registry.FailoverResult) {
+				r.eventf(ctx, execute, corev1.EventTypeWarning, operatorv1alpha1.EventRegistryFailover,
+					"%s: registry %s unreachable, failed over to %s (digest: %s)",
+					name, res.FailedImage, res.Selected.Image, res.Selected.Digest)
+			},
+			func(name operatorv1alpha1.ComponentName, err error) {
+				r.eventf(ctx, execute, corev1.EventTypeWarning, operatorv1alpha1.EventRegistryInvalid,
+					"%s: %v", name, err)
+			},
+		)
+		if !registryOK {
+			now := metav1.Now()
+			execute.Status.Phase = operatorv1alpha1.ExecutePhaseFailed
+			execute.Status.CompletionTime = &now
+			execute.Status.ObservedGeneration = execute.Generation
+			setCondition(&execute.Status.Conditions, metav1.Condition{
+				Type:    operatorv1alpha1.CondRegistrySelected,
+				Status:  metav1.ConditionFalse,
+				Reason:  operatorv1alpha1.ReasonRegistryInvalid.String(),
+				Message: "no valid registry available for one or more components",
+			})
+			if err := r.Status().Update(ctx, execute); err != nil {
+				return &ctrl.Result{}, wrapStatusUpdate(err, "updating status to failed")
+			}
+			r.eventf(ctx, execute, corev1.EventTypeWarning, operatorv1alpha1.EventUpgradeFailed,
+				"Registry validation failed for operationId %q", execute.Spec.OperationID)
+			return &ctrl.Result{}, errx.WithHints(
+				errx.WithReason(errReconcile.New("no valid registry for one or more components"), ReasonNoValidRegistry),
+				"Confirm at least one configured registry serves a valid image for each component")
+		}
+		execute.Status.SelectedSources = updated
+		setCondition(&execute.Status.Conditions, metav1.Condition{
+			Type:    operatorv1alpha1.CondRegistrySelected,
+			Status:  metav1.ConditionTrue,
+			Reason:  operatorv1alpha1.ReasonSelectionComplete.String(),
+			Message: "all selected registries healthy",
+		})
+	}
+
+	execute.Status.ObservedGeneration = execute.Generation
+	if err := r.Status().Update(ctx, execute); err != nil {
+		if apierrors.IsConflict(err) {
+			return &ctrl.Result{Requeue: true}, nil
+		}
+		return &ctrl.Result{}, wrapStatusUpdate(err, "persisting selected sources")
+	}
+	return nil, nil
+}
+
 // failUpgrade drives the Execute CR to a terminal Failed phase with a
 // NodesUpgraded=False condition and a UpgradeFailed warning event. The reconciler
 // is the sole writer of Execute.status.phase.

internal/controller/orbit_controller.go

@@ -8,6 +8,7 @@ import (
 	"slices"
 	"time"
 
+	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -350,6 +351,7 @@ func (r *OrbitReconciler) deleteOwnedCapsules(ctx context.Context, orbitObj *ope
 }
 
 func (r *OrbitReconciler) updateStatus(ctx context.Context, orbitObj *operatorv1alpha1.Orbit) error {
+	before := orbitObj.Status.DeepCopy()
 	now := metav1.Now()
 
 	if orbitObj.Status.Phase == "" {
@@ -364,26 +366,17 @@ func (r *OrbitReconciler) updateStatus(ctx context.Context, orbitObj *operatorv1
 		}
 	}
 
-	ready := metav1.Condition{
-		Type:               "Ready",
-		Status:             metav1.ConditionTrue,
-		LastTransitionTime: now,
-		Reason:             ReasonReconcileSuccess.String(),
-		Message:            "Orbit is ready",
-	}
-	found := false
-	for i, c := range orbitObj.Status.Conditions {
-		if c.Type == "Ready" {
-			orbitObj.Status.Conditions[i] = ready
-			found = true
-			break
-		}
-	}
-	if !found {
-		orbitObj.Status.Conditions = append(orbitObj.Status.Conditions, ready)
-	}
+	apimeta.SetStatusCondition(&orbitObj.Status.Conditions, metav1.Condition{
+		Type:    "Ready",
+		Status:  metav1.ConditionTrue,
+		Reason:  ReasonReconcileSuccess.String(),
+		Message: "Orbit is ready",
+	})
 	r.setNamespaceAllowed(ctx, orbitObj)
 	orbitObj.Status.ObservedGeneration = orbitObj.Generation
+	if equality.Semantic.DeepEqual(before, &orbitObj.Status) {
+		return nil
+	}
 	return common.WarnOnConflict(ctx, r.Status().Update(ctx, orbitObj))
 }