Add NeuVector pre/post/chart-upgrade validation to rancherupgrade package

actions/charts/longhorn.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 
+	catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1"
 	"github.com/rancher/shepherd/clients/rancher"
 	"github.com/rancher/shepherd/clients/rancher/catalog"
 	shepherdCharts "github.com/rancher/shepherd/extensions/charts"
@@ -100,6 +101,123 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma
 	return err
 }
 
+// UpgradeLonghornChart upgrades the longhorn and longhorn-crd charts to
+// installOptions.Version on installOptions.Cluster and waits for Deployed state.
+// No session cleanup is registered: the uninstall cleanup belongs to the install
+// call of the same session, and upgrade-test sessions must not delete the chart.
+func UpgradeLonghornChart(client *rancher.Client, installOptions *InstallOptions) error {
+	serverSetting, err := client.Management.Setting.ByID(serverURLSettingID)
+	if err != nil {
+		return err
+	}
+
+	registrySetting, err := client.Management.Setting.ByID(defaultRegistrySettingID)
+	if err != nil {
+		return err
+	}
+
+	longhornChartUpgradeActionPayload := &PayloadOpts{
+		InstallOptions:  *installOptions,
+		Name:            LonghornChartName,
+		Namespace:       LonghornNamespace,
+		Host:            serverSetting.Value,
+		DefaultRegistry: registrySetting.Value,
+	}
+
+	chartUpgradeAction := newLonghornChartUpgradeAction(longhornChartUpgradeActionPayload)
+
+	catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID)
+	if err != nil {
+		return err
+	}
+
+	bodyBytes, err := json.Marshal(chartUpgradeAction)
+	if err != nil {
+		return err
+	}
+	err = ChartActionWithRetry(context.TODO(), client, verbUpgrade, longhornChartUpgradeActionPayload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbUpgrade, bodyBytes))
+	if err != nil {
+		return err
+	}
+
+	adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session)
+	if err != nil {
+		return err
+	}
+
+	adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID)
+	if err != nil {
+		return err
+	}
+
+	watchAppInterface, err := adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{
+		FieldSelector:  "metadata.name=" + LonghornChartName,
+		TimeoutSeconds: &defaults.WatchTimeoutSeconds,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) {
+		app := event.Object.(*catalogv1.App)
+
+		state := app.Status.Summary.State
+		if state == string(catalogv1.StatusPendingUpgrade) {
+			return true, nil
+		}
+
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	watchAppInterface, err = adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{
+		FieldSelector:  "metadata.name=" + LonghornChartName,
+		TimeoutSeconds: &defaults.WatchTimeoutSeconds,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) {
+		app := event.Object.(*catalogv1.App)
+
+		state := app.Status.Summary.State
+		if state == string(catalogv1.StatusDeployed) {
+			return true, nil
+		}
+
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	return shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{})
+}
+
+// newLonghornChartUpgradeAction is a private helper function that returns the chart upgrade action for Longhorn.
+func newLonghornChartUpgradeAction(p *PayloadOpts) *types.ChartUpgradeAction {
+	// Pass the deletingConfirmationFlag value on the main chart so the upgrade does
+	// not reset it; the CRD chart takes no Longhorn settings (same split as install).
+	chartUpgrade := NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, enableDeletionSetting)
+	chartUpgradeCRD := NewChartUpgrade(p.Name+"-crd", p.Name+"-crd", p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, nil)
+	chartUpgrades := []types.ChartUpgrade{*chartUpgradeCRD, *chartUpgrade}
+
+	chartUpgradeAction := NewChartUpgradeAction(p.Namespace, chartUpgrades)
+	// Disable OpenAPI validation to avoid schema-conflict failures on the large
+	// Longhorn CRDs, which frequently change shape between versions.
+	chartUpgradeAction.DisableOpenAPIValidation = true
+
+	return chartUpgradeAction
+}
+
 // UninstallLonghornChart removes Longhorn from the cluster related to the received catalog client object.
 func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID string, rancherHostname string) error {
 	catalogClient, err := client.GetClusterCatalogClient(clusterID)

actions/charts/neuvector.go

@@ -5,13 +5,18 @@ import (
 	"encoding/json"
 	"time"
 
+	catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1"
 	"github.com/rancher/shepherd/clients/rancher"
 	"github.com/rancher/shepherd/clients/rancher/catalog"
+	shepherdCharts "github.com/rancher/shepherd/extensions/charts"
 	"github.com/rancher/shepherd/extensions/clusters"
+	"github.com/rancher/shepherd/extensions/defaults"
 	"github.com/rancher/shepherd/pkg/api/steve/catalog/types"
+	"github.com/rancher/shepherd/pkg/wait"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kwait "k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
 )
 
 const (
@@ -178,6 +183,137 @@ func InstallNeuVectorChart(client *rancher.Client, payload PayloadOpts) error {
 	return nil
 }
 
+// UpgradeNeuVectorChart upgrades the neuvector and neuvector-crd charts to
+// installOptions.Version on installOptions.Cluster, waits for the Deployed state
+// and for the NeuVector workloads (controller deployment, enforcer daemonset) to
+// become Ready again.
+//
+// The upgrade carries over the values of the installed release (the App resource's
+// spec.values, equivalent to helm --reuse-values) instead of applying a blind
+// version bump: registry overrides required by airgap environments and the
+// enforcer hardening or k3s options set at install time must survive the upgrade,
+// or the upgraded NeuVector would come back with chart defaults (docker.io
+// images, unhardened enforcer) and break those environments. The neuvector-crd
+// chart takes no values, mirroring the install split.
+//
+// No session cleanup is registered: the uninstall cleanup belongs to the install
+// call of the same session, and upgrade-test sessions must not delete the chart.
+func UpgradeNeuVectorChart(client *rancher.Client, installOptions *InstallOptions) error {
+	serverSetting, err := client.Management.Setting.ByID(serverURLSettingID)
+	if err != nil {
+		return err
+	}
+
+	registrySetting, err := client.Management.Setting.ByID(defaultRegistrySettingID)
+	if err != nil {
+		return err
+	}
+
+	catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID)
+	if err != nil {
+		return err
+	}
+
+	// Reading the installed App yields exactly the values overridden at install
+	// time (chart defaults are not stored), which is what must be re-applied.
+	installedApp, err := catalogClient.Apps(NeuVectorNamespace).Get(context.TODO(), NeuVectorChartName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	neuVectorChartUpgradeActionPayload := &PayloadOpts{
+		InstallOptions:  *installOptions,
+		Name:            NeuVectorChartName,
+		Namespace:       NeuVectorNamespace,
+		Host:            serverSetting.Value,
+		DefaultRegistry: registrySetting.Value,
+	}
+
+	chartUpgradeAction := newNeuVectorChartUpgradeAction(neuVectorChartUpgradeActionPayload, map[string]interface{}(installedApp.Spec.Values))
+
+	bodyBytes, err := json.Marshal(chartUpgradeAction)
+	if err != nil {
+		return err
+	}
+	err = ChartActionWithRetry(context.TODO(), client, verbUpgrade, neuVectorChartUpgradeActionPayload, catalog.RancherChartRepo, []string{NeuVectorChartName, NeuVectorChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbUpgrade, bodyBytes))
+	if err != nil {
+		return err
+	}
+
+	adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session)
+	if err != nil {
+		return err
+	}
+
+	adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID)
+	if err != nil {
+		return err
+	}
+
+	watchAppInterface, err := adminCatalogClient.Apps(NeuVectorNamespace).Watch(context.TODO(), metav1.ListOptions{
+		FieldSelector:  "metadata.name=" + NeuVectorChartName,
+		TimeoutSeconds: &defaults.WatchTimeoutSeconds,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) {
+		app := event.Object.(*catalogv1.App)
+
+		state := app.Status.Summary.State
+		if state == string(catalogv1.StatusPendingUpgrade) {
+			return true, nil
+		}
+
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	watchAppInterface, err = adminCatalogClient.Apps(NeuVectorNamespace).Watch(context.TODO(), metav1.ListOptions{
+		FieldSelector:  "metadata.name=" + NeuVectorChartName,
+		TimeoutSeconds: &defaults.WatchTimeoutSeconds,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) {
+		app := event.Object.(*catalogv1.App)
+
+		state := app.Status.Summary.State
+		if state == string(catalogv1.StatusDeployed) {
+			return true, nil
+		}
+
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, NeuVectorNamespace, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	return shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, NeuVectorNamespace, metav1.ListOptions{})
+}
+
+// newNeuVectorChartUpgradeAction is a private helper function that returns the
+// chart upgrade action for NeuVector, reusing the installed release values on the
+// main chart. OpenAPI validation stays enabled: the neuvector CRDs are small and
+// do not hit the schema-conflict failures seen on large CRD charts.
+func newNeuVectorChartUpgradeAction(p *PayloadOpts, installedValues map[string]interface{}) *types.ChartUpgradeAction {
+	chartUpgrade := NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, installedValues)
+	chartUpgradeCRD := NewChartUpgrade(p.Name+"-crd", p.Name+"-crd", p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, nil)
+	chartUpgrades := []types.ChartUpgrade{*chartUpgradeCRD, *chartUpgrade}
+
+	return NewChartUpgradeAction(p.Namespace, chartUpgrades)
+}
+
 func uninstallNeuVectorChart(client *rancher.Client, namespace string, clusterID string) error {
 	catalogClient, err := client.GetClusterCatalogClient(clusterID)
 	if err != nil {

actions/clusters/local.go

@@ -1,13 +1,18 @@
 package clusters
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/Masterminds/semver/v3"
 	"github.com/rancher/shepherd/clients/rancher"
+	"github.com/rancher/shepherd/extensions/defaults"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
@@ -37,6 +42,41 @@ func IsRancherVersionAbove(client *rancher.Client, minVersion string) (bool, err
 	return serverVersion.GreaterThan(minimumVersion), nil
 }
 
+// WaitRancherVersion polls the management server-version setting until it equals
+// targetVersion (semver-compared) or timeout elapses. Gates post-upgrade assertions
+// on Rancher server convergence; transient read errors during the upgrade rollout
+// are retried, not returned.
+func WaitRancherVersion(client *rancher.Client, targetVersion string, timeout time.Duration) error {
+	target, err := parseRancherVersion(targetVersion)
+	if err != nil {
+		return err
+	}
+
+	return wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, timeout, true, func(ctx context.Context) (done bool, err error) {
+		serverVersionRawValue, err := client.Management.Setting.ByID(serverVersionSetting)
+		if err != nil {
+			// The server may be mid-restart during the upgrade rollout; retry the read.
+			// Logged so a terminal cause (e.g. a revoked token or unreachable API)
+			// is visible in the log instead of surfacing only as the final timeout.
+			logrus.Debugf("server-version read failed, retrying: %v", err)
+			return false, nil
+		}
+
+		serverVersion, err := parseRancherVersion(serverVersionRawValue.Value)
+		if err != nil {
+			logrus.Debugf("server-version value %q not parseable yet, retrying: %v", serverVersionRawValue.Value, err)
+			return false, nil
+		}
+
+		if serverVersion.Equal(target) {
+			return true, nil
+		}
+
+		logrus.Debugf("Rancher server version is %q, waiting for %q", serverVersionRawValue.Value, targetVersion)
+		return false, nil
+	})
+}
+
 func parseRancherVersion(versionInput string) (*semver.Version, error) {
 	trimmedVersion := strings.TrimSpace(strings.TrimPrefix(versionInput, "v"))
 	if trimmedVersion == "" {