Generalize rancher-upgrade pipeline to a TEST_MODULE selector

actions/charts/longhorn.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 
+	catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1"
 	"github.com/rancher/shepherd/clients/rancher"
 	"github.com/rancher/shepherd/clients/rancher/catalog"
 	shepherdCharts "github.com/rancher/shepherd/extensions/charts"
@@ -100,6 +101,123 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma
 	return err
 }
 
+// UpgradeLonghornChart upgrades the longhorn and longhorn-crd charts to
+// installOptions.Version on installOptions.Cluster and waits for Deployed state.
+// No session cleanup is registered: the uninstall cleanup belongs to the install
+// call of the same session, and upgrade-test sessions must not delete the chart.
+func UpgradeLonghornChart(client *rancher.Client, installOptions *InstallOptions) error {
+	serverSetting, err := client.Management.Setting.ByID(serverURLSettingID)
+	if err != nil {
+		return err
+	}
+
+	registrySetting, err := client.Management.Setting.ByID(defaultRegistrySettingID)
+	if err != nil {
+		return err
+	}
+
+	longhornChartUpgradeActionPayload := &PayloadOpts{
+		InstallOptions:  *installOptions,
+		Name:            LonghornChartName,
+		Namespace:       LonghornNamespace,
+		Host:            serverSetting.Value,
+		DefaultRegistry: registrySetting.Value,
+	}
+
+	chartUpgradeAction := newLonghornChartUpgradeAction(longhornChartUpgradeActionPayload)
+
+	catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID)
+	if err != nil {
+		return err
+	}
+
+	bodyBytes, err := json.Marshal(chartUpgradeAction)
+	if err != nil {
+		return err
+	}
+	err = ChartActionWithRetry(context.TODO(), client, verbUpgrade, longhornChartUpgradeActionPayload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbUpgrade, bodyBytes))
+	if err != nil {
+		return err
+	}
+
+	adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session)
+	if err != nil {
+		return err
+	}
+
+	adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID)
+	if err != nil {
+		return err
+	}
+
+	watchAppInterface, err := adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{
+		FieldSelector:  "metadata.name=" + LonghornChartName,
+		TimeoutSeconds: &defaults.WatchTimeoutSeconds,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) {
+		app := event.Object.(*catalogv1.App)
+
+		state := app.Status.Summary.State
+		if state == string(catalogv1.StatusPendingUpgrade) {
+			return true, nil
+		}
+
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	watchAppInterface, err = adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{
+		FieldSelector:  "metadata.name=" + LonghornChartName,
+		TimeoutSeconds: &defaults.WatchTimeoutSeconds,
+	})
+	if err != nil {
+		return err
+	}
+
+	err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) {
+		app := event.Object.(*catalogv1.App)
+
+		state := app.Status.Summary.State
+		if state == string(catalogv1.StatusDeployed) {
+			return true, nil
+		}
+
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	return shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{})
+}
+
+// newLonghornChartUpgradeAction is a private helper function that returns the chart upgrade action for Longhorn.
+func newLonghornChartUpgradeAction(p *PayloadOpts) *types.ChartUpgradeAction {
+	// Pass the deletingConfirmationFlag value on the main chart so the upgrade does
+	// not reset it; the CRD chart takes no Longhorn settings (same split as install).
+	chartUpgrade := NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, enableDeletionSetting)
+	chartUpgradeCRD := NewChartUpgrade(p.Name+"-crd", p.Name+"-crd", p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, nil)
+	chartUpgrades := []types.ChartUpgrade{*chartUpgradeCRD, *chartUpgrade}
+
+	chartUpgradeAction := NewChartUpgradeAction(p.Namespace, chartUpgrades)
+	// Disable OpenAPI validation to avoid schema-conflict failures on the large
+	// Longhorn CRDs, which frequently change shape between versions.
+	chartUpgradeAction.DisableOpenAPIValidation = true
+
+	return chartUpgradeAction
+}
+
 // UninstallLonghornChart removes Longhorn from the cluster related to the received catalog client object.
 func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID string, rancherHostname string) error {
 	catalogClient, err := client.GetClusterCatalogClient(clusterID)

actions/clusters/local.go

@@ -1,13 +1,18 @@
 package clusters
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/Masterminds/semver/v3"
 	"github.com/rancher/shepherd/clients/rancher"
+	"github.com/rancher/shepherd/extensions/defaults"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
@@ -37,6 +42,41 @@ func IsRancherVersionAbove(client *rancher.Client, minVersion string) (bool, err
 	return serverVersion.GreaterThan(minimumVersion), nil
 }
 
+// WaitRancherVersion polls the management server-version setting until it equals
+// targetVersion (semver-compared) or timeout elapses. Gates post-upgrade assertions
+// on Rancher server convergence; transient read errors during the upgrade rollout
+// are retried, not returned.
+func WaitRancherVersion(client *rancher.Client, targetVersion string, timeout time.Duration) error {
+	target, err := parseRancherVersion(targetVersion)
+	if err != nil {
+		return err
+	}
+
+	return wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, timeout, true, func(ctx context.Context) (done bool, err error) {
+		serverVersionRawValue, err := client.Management.Setting.ByID(serverVersionSetting)
+		if err != nil {
+			// The server may be mid-restart during the upgrade rollout; retry the read.
+			// Logged so a terminal cause (e.g. a revoked token or unreachable API)
+			// is visible in the log instead of surfacing only as the final timeout.
+			logrus.Debugf("server-version read failed, retrying: %v", err)
+			return false, nil
+		}
+
+		serverVersion, err := parseRancherVersion(serverVersionRawValue.Value)
+		if err != nil {
+			logrus.Debugf("server-version value %q not parseable yet, retrying: %v", serverVersionRawValue.Value, err)
+			return false, nil
+		}
+
+		if serverVersion.Equal(target) {
+			return true, nil
+		}
+
+		logrus.Debugf("Rancher server version is %q, waiting for %q", serverVersionRawValue.Value, targetVersion)
+		return false, nil
+	})
+}
+
 func parseRancherVersion(versionInput string) (*semver.Version, error) {
 	trimmedVersion := strings.TrimSpace(strings.TrimPrefix(versionInput, "v"))
 	if trimmedVersion == "" {