diff --git a/actions/charts/longhorn.go b/actions/charts/longhorn.go index 89c8640cb3..c1191e3859 100644 --- a/actions/charts/longhorn.go +++ b/actions/charts/longhorn.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" "github.com/rancher/shepherd/clients/rancher" "github.com/rancher/shepherd/clients/rancher/catalog" shepherdCharts "github.com/rancher/shepherd/extensions/charts" @@ -100,6 +101,123 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma return err } +// UpgradeLonghornChart upgrades the longhorn and longhorn-crd charts to +// installOptions.Version on installOptions.Cluster and waits for Deployed state. +// No session cleanup is registered: the uninstall cleanup belongs to the install +// call of the same session, and upgrade-test sessions must not delete the chart. +func UpgradeLonghornChart(client *rancher.Client, installOptions *InstallOptions) error { + serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) + if err != nil { + return err + } + + registrySetting, err := client.Management.Setting.ByID(defaultRegistrySettingID) + if err != nil { + return err + } + + longhornChartUpgradeActionPayload := &PayloadOpts{ + InstallOptions: *installOptions, + Name: LonghornChartName, + Namespace: LonghornNamespace, + Host: serverSetting.Value, + DefaultRegistry: registrySetting.Value, + } + + chartUpgradeAction := newLonghornChartUpgradeAction(longhornChartUpgradeActionPayload) + + catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + bodyBytes, err := json.Marshal(chartUpgradeAction) + if err != nil { + return err + } + err = ChartActionWithRetry(context.TODO(), client, verbUpgrade, longhornChartUpgradeActionPayload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbUpgrade, bodyBytes)) + if err != nil { + return err + } + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + if err != nil { + return err + } + + adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + watchAppInterface, err := adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + LonghornChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusPendingUpgrade) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + watchAppInterface, err = adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + LonghornChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusDeployed) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{}) + if err != nil { + return err + } + + return shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{}) +} + +// newLonghornChartUpgradeAction is a private helper function that returns the chart upgrade action for Longhorn. +func newLonghornChartUpgradeAction(p *PayloadOpts) *types.ChartUpgradeAction { + // Pass the deletingConfirmationFlag value on the main chart so the upgrade does + // not reset it; the CRD chart takes no Longhorn settings (same split as install). + chartUpgrade := NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, enableDeletionSetting) + chartUpgradeCRD := NewChartUpgrade(p.Name+"-crd", p.Name+"-crd", p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, nil) + chartUpgrades := []types.ChartUpgrade{*chartUpgradeCRD, *chartUpgrade} + + chartUpgradeAction := NewChartUpgradeAction(p.Namespace, chartUpgrades) + // Disable OpenAPI validation to avoid schema-conflict failures on the large + // Longhorn CRDs, which frequently change shape between versions. + chartUpgradeAction.DisableOpenAPIValidation = true + + return chartUpgradeAction +} + // UninstallLonghornChart removes Longhorn from the cluster related to the received catalog client object. func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID string, rancherHostname string) error { catalogClient, err := client.GetClusterCatalogClient(clusterID) diff --git a/actions/clusters/local.go b/actions/clusters/local.go index 206e523b81..16940dfd53 100644 --- a/actions/clusters/local.go +++ b/actions/clusters/local.go @@ -1,13 +1,18 @@ package clusters import ( + "context" "errors" "fmt" "regexp" "strings" + "time" "github.com/Masterminds/semver/v3" "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -37,6 +42,41 @@ func IsRancherVersionAbove(client *rancher.Client, minVersion string) (bool, err return serverVersion.GreaterThan(minimumVersion), nil } +// WaitRancherVersion polls the management server-version setting until it equals +// targetVersion (semver-compared) or timeout elapses. Gates post-upgrade assertions +// on Rancher server convergence; transient read errors during the upgrade rollout +// are retried, not returned. +func WaitRancherVersion(client *rancher.Client, targetVersion string, timeout time.Duration) error { + target, err := parseRancherVersion(targetVersion) + if err != nil { + return err + } + + return wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, timeout, true, func(ctx context.Context) (done bool, err error) { + serverVersionRawValue, err := client.Management.Setting.ByID(serverVersionSetting) + if err != nil { + // The server may be mid-restart during the upgrade rollout; retry the read. + // Logged so a terminal cause (e.g. a revoked token or unreachable API) + // is visible in the log instead of surfacing only as the final timeout. + logrus.Debugf("server-version read failed, retrying: %v", err) + return false, nil + } + + serverVersion, err := parseRancherVersion(serverVersionRawValue.Value) + if err != nil { + logrus.Debugf("server-version value %q not parseable yet, retrying: %v", serverVersionRawValue.Value, err) + return false, nil + } + + if serverVersion.Equal(target) { + return true, nil + } + + logrus.Debugf("Rancher server version is %q, waiting for %q", serverVersionRawValue.Value, targetVersion) + return false, nil + }) +} + func parseRancherVersion(versionInput string) (*semver.Version, error) { trimmedVersion := strings.TrimSpace(strings.TrimPrefix(versionInput, "v")) if trimmedVersion == "" { diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go new file mode 100644 index 0000000000..e7da7d5b29 --- /dev/null +++ b/actions/storage/checksum.go @@ -0,0 +1,221 @@ +package storage + +import ( + "context" + "crypto/sha256" + "fmt" + "regexp" + "strings" + + "github.com/rancher/shepherd/clients/rancher" + steveV1 "github.com/rancher/shepherd/clients/rancher/v1" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/extensions/defaults/namespaces" + "github.com/rancher/shepherd/extensions/defaults/stevetypes" + "github.com/rancher/shepherd/extensions/kubeapi/cluster" + "github.com/rancher/shepherd/extensions/kubeconfig" + "github.com/rancher/tests/actions/kubeapi/volumes/persistentvolumeclaims" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" +) + +// ChecksumDataFile is the filename written under MountPath by CreateChecksumWorkload. +const ChecksumDataFile = "checksum-data" + +// checksumContentPattern constrains the content accepted by CreateChecksumWorkload: +// content is interpolated into a single-quoted sh -c command, so a value outside +// alphanumeric/hyphen (notably a quote) would break the quoting. Enforced in code +// so a future caller cannot silently inject shell. +var checksumContentPattern = regexp.MustCompile(`^[A-Za-z0-9-]+$`) + +// CreateChecksumWorkload creates a 1GiB PVC named on storageClassName in the +// default namespace, mounts it into a single-replica nginx deployment named at +// MountPath, waits for rollout, then writes content to MountPath/ChecksumDataFile via +// kubectl exec. Deterministic names let a later test phase (running as a separate +// process) relocate the resources. content must be shell-safe (alphanumeric/hyphen); +// it is interpolated into an sh -c command. No session cleanup is registered: callers +// are upgrade tests whose resources must outlive the process. +func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageClassName string, name string, content string) error { + if !checksumContentPattern.MatchString(content) { + return fmt.Errorf("content %q must match %s: it is interpolated into a single-quoted sh -c command", + content, checksumContentPattern.String()) + } + + storageClass, err := GetStorageClass(client, clusterID, storageClassName) + if err != nil { + return err + } + + wrangler, err := cluster.GetClusterWranglerContext(client, clusterID) + if err != nil { + return err + } + + accessModes := []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + } + + // Deliberate deviation from CreatePVC: no PV-delete/Longhorn-volume cleanup funcs + // are registered, because these resources must survive into later upgrade phases. + _, err = persistentvolumeclaims.CreatePersistentVolumeClaim( + client, + clusterID, + name, + "test-pvc-volume", + namespaces.Default, + 1, + accessModes, + nil, + &storageClass, + ) + if err != nil { + return err + } + + pvc := &corev1.PersistentVolumeClaim{} + // TenMinuteTimeout: the first volume on a freshly installed Longhorn waits + // on engine-image pull, instance-manager startup, and CSI registration, + // which routinely exceeds a minute on a cold cluster. + err = wait.PollUntilContextTimeout(context.Background(), pollInterval, defaults.TenMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + pvc, err = wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + return pvc.Status.Phase == persistentvolumeclaims.PersistentVolumeBoundStatus, nil + }) + if err != nil { + return err + } + logrus.Debugf("PVC %s is Bound to volume %s", pvc.Name, pvc.Spec.VolumeName) + + // Shared nginx+PVC deployment construction with CreatePVCWorkload; the + // deterministic name makes PVC, volume mount, deployment, and relocation + // label all equal name, so later phases re-find the workload from a + // separate process. + _, err = createNginxDeploymentWithPVC(client, clusterID, name, name, name) + if err != nil { + return err + } + + pod, err := getChecksumPod(client, clusterID, name) + if err != nil { + return err + } + + restConfig, err := getDownstreamRestConfig(client, clusterID) + if err != nil { + return err + } + + writeCommand := []string{"sh", "-c", "printf '%s' '" + content + "' > " + MountPath + "/" + ChecksumDataFile} + logrus.Debugf("Writing %s/%s on pod %s", MountPath, ChecksumDataFile, pod.Name) + _, err = kubeconfig.KubectlExec(restConfig, pod.Name, namespaces.Default, writeCommand) + + return err +} + +// VerifyChecksumWorkload relocates the PVC and pod created by CreateChecksumWorkload, +// asserts the PVC is still Bound, and asserts the sha256 of MountPath/ChecksumDataFile +// inside the running pod equals sha256(content). +func VerifyChecksumWorkload(client *rancher.Client, clusterID string, name string, content string) error { + wrangler, err := cluster.GetClusterWranglerContext(client, clusterID) + if err != nil { + return err + } + + pvc, err := wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) + if err != nil { + return err + } + + if pvc.Status.Phase != persistentvolumeclaims.PersistentVolumeBoundStatus { + return fmt.Errorf("PVC %s is %s, expected %s", name, pvc.Status.Phase, persistentvolumeclaims.PersistentVolumeBoundStatus) + } + + pod, err := getChecksumPod(client, clusterID, name) + if err != nil { + return err + } + + restConfig, err := getDownstreamRestConfig(client, clusterID) + if err != nil { + return err + } + + checksumCommand := []string{"sha256sum", MountPath + "/" + ChecksumDataFile} + output, err := kubeconfig.KubectlExec(restConfig, pod.Name, namespaces.Default, checksumCommand) + if err != nil { + return err + } + + // The exec runs with a TTY, so the output carries \r line endings; strip them + // before extracting the hash (first whitespace-separated field). + outputFields := strings.Fields(strings.ReplaceAll(output.String(), "\r", "")) + if len(outputFields) == 0 { + return fmt.Errorf("empty sha256sum output on pod %s", pod.Name) + } + + actualChecksum := outputFields[0] + expectedChecksum := fmt.Sprintf("%x", sha256.Sum256([]byte(content))) + if actualChecksum != expectedChecksum { + return fmt.Errorf("checksum mismatch on pod %s: expected %s, got %s", pod.Name, expectedChecksum, actualChecksum) + } + + return nil +} + +// getChecksumPod returns the pod in the default namespace labeled with +// DeploymentIdentifierLabel == name, waiting until it exists and is Running. +func getChecksumPod(client *rancher.Client, clusterID string, name string) (*steveV1.SteveAPIObject, error) { + var checksumPod *steveV1.SteveAPIObject + err := wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, defaults.TwoMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + steveclient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return false, err + } + + podsResp, err := steveclient.SteveType(stevetypes.Pod).NamespacedSteveClient(namespaces.Default).List(nil) + if err != nil { + return false, err + } + + for i := range podsResp.Data { + pod := &podsResp.Data[i] + if pod.Labels[DeploymentIdentifierLabel] != name { + continue + } + + podStatus := &corev1.PodStatus{} + err = steveV1.ConvertToK8sType(pod.Status, podStatus) + if err != nil { + return false, err + } + + if podStatus.Phase != corev1.PodRunning { + logrus.Debugf("Pod %s is %s, waiting for Running", pod.Name, podStatus.Phase) + return false, nil + } + + checksumPod = pod + return true, nil + } + + return false, nil + }) + + return checksumPod, err +} + +// getDownstreamRestConfig builds a rest config for kubectl exec against the cluster. +func getDownstreamRestConfig(client *rancher.Client, clusterID string) (*rest.Config, error) { + kubeConfig, err := kubeconfig.GetKubeconfig(client, clusterID) + if err != nil { + return nil, err + } + + return (*kubeConfig).ClientConfig() +} diff --git a/actions/storage/pvc.go b/actions/storage/pvc.go index 163c0df025..4030e6823d 100644 --- a/actions/storage/pvc.go +++ b/actions/storage/pvc.go @@ -158,7 +158,6 @@ func CreatePVC(client *rancher.Client, clusterID string, storageClassName string } return false, err }) - if err != nil { return nil, "", err } @@ -185,19 +184,25 @@ func CreatePVCWorkload(t *testing.T, client *rancher.Client, clusterID string, s persistentVolumeClaim, volumeName, err := CreatePVC(client, clusterID, storageClassName) require.NoError(t, err) - logrus.Debugf("Creating Nginx deployment with %s PVC (volume name %s)", persistentVolumeClaim.Name, volumeName) - nginxResponse, err := createNginxDeploymentWithPVC(client, clusterID, persistentVolumeClaim.Name, volumeName) + // Random workload name: this helper targets single-process functional tests; + // callers needing a deterministic (cross-phase relocatable) name call + // createNginxDeploymentWithPVC directly (see CreateChecksumWorkload). + deploymentName := namegenerator.AppendRandomString(containerNamePrefix) + logrus.Debugf("Creating Nginx deployment %s with %s PVC (volume name %s)", deploymentName, persistentVolumeClaim.Name, volumeName) + nginxResponse, err := createNginxDeploymentWithPVC(client, clusterID, persistentVolumeClaim.Name, volumeName, deploymentName) require.NoError(t, err) return nginxResponse } -// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace -func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcName, volName string) (*steveV1.SteveAPIObject, error) { +// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace. +// The deployment and its DeploymentIdentifierLabel both use deploymentName, letting callers +// pick a random name (functional tests) or a deterministic name (workloads a later test +// phase must re-find from a separate process). +func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcName, volName, deploymentName string) (*steveV1.SteveAPIObject, error) { logrus.Tracef("Vol: %s", volName) logrus.Tracef("Pod: %s", pvcName) - containerName := namegenerator.AppendRandomString(containerNamePrefix) volMount := &corev1.VolumeMount{ MountPath: MountPath, Name: volName, @@ -218,8 +223,8 @@ func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcN } containerTemplate := wloads.NewContainer(nginxName, nginxName, corev1.PullAlways, []corev1.VolumeMount{*volMount}, []corev1.EnvFromSource{}, nil, nil, nil) - podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: containerName}, nil) - deployment := wloads.NewDeploymentTemplate(containerName, namespaces.Default, podTemplate, true, nil) + podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: deploymentName}, nil) + deployment := wloads.NewDeploymentTemplate(deploymentName, namespaces.Default, podTemplate, true, nil) deploymentResp, err := steveclient.SteveType(stevetypes.Deployment).Create(deployment) if err != nil { @@ -227,7 +232,7 @@ func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcN } err = charts.WatchAndWaitDeployments(client, clusterID, namespaces.Default, metav1.ListOptions{ - FieldSelector: "metadata.name=" + containerName, + FieldSelector: "metadata.name=" + deploymentName, }) return deploymentResp, err diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e new file mode 100755 index 0000000000..1c72ba46c7 --- /dev/null +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -0,0 +1,1228 @@ +#!groovy +/** + * Dedicated Rancher server upgrade validation pipeline (rancher/tests#968, PRD #946). + * + * Six-stage flow, community leg: + * 1. Deploys a 2.14-latest environment via qa-infra-automation + * (make backend-s3, workspace-new, infra-up, cluster, rancher) + * 2. Pre-upgrade phase: installs the TEST_MODULE chart + seeds its upgrade + * state (validation/rancherupgrade, UpgradeTestSuite/*PreUpgrade) + * 3. Upgrades the Rancher server in place via `make rancher-upgrade` + * and re-injects the rotated admin token into the test config + * 4. Post-upgrade phase: asserts server version and module state survival + * 5. Chart-upgrade phase: upgrades the module chart to the version offered + * by the upgraded Rancher's catalog and re-verifies (skipped for fleet: + * Fleet ships inside Rancher with no standalone chart) + * 6. Always destroys the environment (infra-down AUTO_APPROVE=yes) + * + * Versions resolve at runtime when RANCHER_VERSION_FROM / RANCHER_VERSION_TO_UPGRADE + * are empty (qa-infra-automation `make rancher-resolve-version`); pins override. + * + * Structurally cloned from validation/Jenkinsfile.e2e (deploy flow, admin-token + * injection, Qase testrun lifecycle, destroy post) and validation/pipeline/ + * Jenkinsfile.airgap-rke2-tests (make.runTarget call shapes, teardown guard). + * Unlike Jenkinsfile.e2e, this pipeline has no parallel downstream fan-out: the + * three test phases run sequentially against one shared Qase test run because + * each phase depends on the previous one's in-cluster state. + * + * Consumes shared functions from qa-jenkins-library: + * make.runTarget (backend-s3, workspace-new, infra-up, cluster, rancher, + * rancher-resolve-version, rancher-upgrade, infra-down) + * airgap.standardCheckout + * infrastructure.parseAndSubstituteVars, infrastructure.writeConfig, + * infrastructure.generateWorkspaceName, infrastructure.archiveWorkspaceName, + * infrastructure.writeSshKey, infrastructure.detectPublicIp + * property.useWithProperties + * + * Admin token generation uses a direct ansible-playbook call inside the + * rancher-go-test image (no make token target exists upstream yet). + */ + +def libraryBranch = env.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +library "qa-jenkins-library@${libraryBranch}" + +def DEFAULT_RANCHER_PASSWORD = 'X%S3bPyg@n8kWcRs675l#fH0' + +// Build-scoped image tags: every direct docker invocation below runs the +// pinned tag so this job never depends on (or mutates) whatever some other +// job last built under the shared :latest tag on the same agent. The :latest +// alias is still applied at build time because qa-jenkins-library's +// make.runTarget resolves that name for its own container runs. +def INFRA_TOOLS_IMAGE = "rancher-infra-tools:pitupgrade-${env.BUILD_NUMBER}" +def GO_TEST_IMAGE = "rancher-go-test:pitupgrade-${env.BUILD_NUMBER}" + +// Suite/method selectors passed as GOTEST_TESTCASE to the downstream executor. +// Each phase is its own go test invocation (-run regex), so cross-phase state +// lives in-cluster under deterministic names, never in process memory. +// TEST_MODULE selects the entry (checkout stage fails fast on an unknown name). +// fleet ships inside Rancher with no standalone chart, so its map has no chart +// phase and the chart-upgrade stage is skipped for it. +def UPGRADE_TEST_MODULES = [ + longhorn: [ + pre: '-run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade"', + post: '-run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade"', + chart: '-run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade"' + ], + neuvector: [ + pre: '-run "TestNeuVectorUpgradeTestSuite/TestNeuVectorPreUpgrade"', + post: '-run "TestNeuVectorUpgradeTestSuite/TestNeuVectorPostUpgrade"', + chart: '-run "TestNeuVectorUpgradeTestSuite/TestNeuVectorChartUpgrade"' + ], + fleet: [ + pre: '-run "TestFleetUpgradeTestSuite/TestFleetPreUpgrade"', + post: '-run "TestFleetUpgradeTestSuite/TestFleetPostUpgrade"' + ], + observability: [ + pre: '-run "TestObservabilityUpgradeTestSuite/TestObservabilityPreUpgrade"', + post: '-run "TestObservabilityUpgradeTestSuite/TestObservabilityPostUpgrade"', + chart: '-run "TestObservabilityUpgradeTestSuite/TestObservabilityChartUpgrade"' + ], + all: [ + // Every module's suite in one pipeline run; the broad regex also + // picks up modules added to the package later. Fleet has no chart + // subtest, so the chart phase runs its Setup/TearDown with zero + // matched methods (harmless: its config keys are still threaded). + pre: '-run "Test.*UpgradeTestSuite/Test.*PreUpgrade"', + post: '-run "Test.*UpgradeTestSuite/Test.*PostUpgrade"', + chart: '-run "Test.*UpgradeTestSuite/Test.*ChartUpgrade"' + ] +] + +// Binding variable (no def) so every stage's script/when block sees it; set in +// the Checkout stage, read by the three phase stages below. +TEST_PHASES = null + +/** + * Shared teardown helper: re-renders tfvars from params, tears down tofu workspace + * via make infra-down AUTO_APPROVE=yes. + * Sets env.INFRA_CLEANED to 'true' on success. + */ +def destroyInfrastructure(String reason) { + def wsName = '' + try { + if (fileExists('workspace_name.txt')) { + wsName = readFile('workspace_name.txt')?.trim() + } + } catch (e) { + echo "Could not read workspace_name.txt: ${e.message}" + } + + if (!wsName || !env.INFRA_DIR || env.INFRA_CLEANED == 'true') { + echo "Skipping teardown (${reason}): no workspace, no infra dir, or already cleaned" + return + } + + try { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + // Re-render terraform.tfvars so tofu destroy can access AWS state + def tofuDir = "${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes" + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME, + 'RUNNER_PUBLIC_IP': env.RUNNER_PUBLIC_IP + ] + ) + infrastructure.writeConfig( + path: "${tofuDir}/terraform.tfvars", + content: terraformConfig + ) + make.runTarget(target: 'infra-down', dir: env.INFRA_DIR, makeArgs: "WORKSPACE=${wsName} AUTO_APPROVE=yes") + env.INFRA_CLEANED = 'true' + } + } catch (cleanupErr) { + echo "Teardown failed (${reason}): ${cleanupErr.message}" + } +} + +/** + * Failure-only diagnostics: capture cluster state through the rancher-infra-tools + * container (kubectl + helm + kubeconfig from the deploy) into workspace files + * and archive them. Best-effort; never masks the failure it reports on. + */ +def collectUpgradeDiagnostics() { + if (!env.INFRA_DIR) { + echo 'Skipping diagnostics: no infra checkout available' + return + } + try { + def workspace = pwd() + sh """ + docker run --rm --platform linux/amd64 \ + -v ${workspace}:/workspace \ + -e KUBECONFIG=/workspace/${env.INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml \ + -w /workspace \ + ${INFRA_TOOLS_IMAGE} \ + sh -c ' + set +e + kubectl get nodes -o wide > diagnostics-nodes.txt 2>&1 + kubectl get pods -A > diagnostics-pods.txt 2>&1 + helm history rancher -n cattle-system > diagnostics-helm-history.txt 2>&1 + kubectl logs -n cattle-system -l app=rancher --tail=300 > diagnostics-rancher-logs.txt 2>&1 + kubectl logs -n cattle-cluster-system -l app=cattle-cluster-agent --tail=300 > diagnostics-agent-logs.txt 2>&1 + exit 0 + ' + """ + if (params.REPORT_ARTIFACTS) { + archiveArtifacts artifacts: 'diagnostics-*.txt', allowEmptyArchive: true + } + } catch (diagErr) { + echo "Diagnostics collection failed (non-fatal): ${diagErr.message}" + } +} + +pipeline { + agent any + + options { + ansiColor('xterm') + // Budget: three sequential test phases (TEST_TIMEOUT each, default 60m) + // plus deploy, server upgrade, and destroy (~90m overhead). Must exceed + // the sum of the stage budgets or a legitimately slow run aborts + // mid-phase; revisit when raising TEST_TIMEOUT. + timeout(time: 300, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '30')) + } + + parameters { + // ── Repository parameters ────────────────────────────── + string( + name: 'QA_JENKINS_LIBRARY_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-jenkins-library to use' + ) + // Repo params keep whatever the job was last triggered with: a + // declarative pipeline re-applies its parameters{} block to the job + // after each run, and hard-coded defaults here would clobber the + // JJB-configured per-controller values (fork + slice branches on the + // staging controller, upstream main on the main controller). + string( + name: 'TESTS_REPO_URL', + defaultValue: params.TESTS_REPO_URL ?: 'https://github.com/rancher/tests', + description: 'URL of rancher/tests repository' + ) + string( + name: 'TESTS_BRANCH', + defaultValue: params.TESTS_BRANCH ?: 'main', + description: 'Branch of rancher/tests repository' + ) + string( + name: 'QA_INFRA_REPO_URL', + defaultValue: params.QA_INFRA_REPO_URL ?: 'https://github.com/rancher/qa-infra-automation', + description: 'URL of qa-infra-automation repository' + ) + string( + name: 'QA_INFRA_BRANCH', + defaultValue: params.QA_INFRA_BRANCH ?: 'main', + description: 'Branch of qa-infra-automation repository' + ) + + // ── Upgrade parameters ───────────────────────────────── + string( + name: 'RANCHER_VERSION_FROM', + defaultValue: '', + description: 'Source Rancher version to deploy (empty = resolve latest 2.14.x patch at runtime)' + ) + string( + name: 'RANCHER_VERSION_TO_UPGRADE', + defaultValue: '', + description: 'Target Rancher version for the server upgrade (empty = resolve latest released final at runtime)' + ) + string( + name: 'RANCHER_CHART_REPO_FLAVOR', + defaultValue: 'community', + description: 'Chart repo flavor (community|prime): selects the chart index for version resolution, the repo the pre-upgrade Rancher is deployed from, and the repo the server upgrade pulls from' + ) + + // ── Infrastructure parameters ────────────────────────── + string( + name: 'HOSTNAME_PREFIX', + defaultValue: 'pitupgrade', + description: 'Hostname prefix for *.qa.rancher.space and other AWS resources' + ) + string( + name: 'RKE2_VERSION', + defaultValue: 'v1.35.1+rke2r1', + description: 'RKE2 version to deploy' + ) + password( + name: 'RANCHER_BOOTSTRAP_PASSWORD', + defaultValue: DEFAULT_RANCHER_PASSWORD, + description: 'Rancher bootstrap password for initial setup' + ) + password( + name: 'RANCHER_ADMIN_PASSWORD', + defaultValue: DEFAULT_RANCHER_PASSWORD, + description: 'Rancher admin password after bootstrap' + ) + booleanParam( + name: 'DESTROY_ON_FAILURE', + defaultValue: true, + description: 'Collect diagnostics and tear down infrastructure if pipeline fails' + ) + booleanParam( + name: 'DESTROY_AFTER_TESTS', + defaultValue: true, + description: 'Tear down infrastructure after a successful run; failed runs are governed by DESTROY_ON_FAILURE' + ) + + // ── Terraform parameters ─────────────────────────────── + string( + name: 'S3_BUCKET_NAME', + defaultValue: 'jenkins-terraform-state-storage', + description: 'S3 bucket name where Terraform state is stored' + ) + string( + name: 'S3_BUCKET_REGION', + defaultValue: 'us-east-2', + description: 'AWS region where the S3 bucket is located' + ) + string( + name: 'S3_KEY_PREFIX', + defaultValue: 'terraform.tfstate', + description: 'S3 key prefix for the Terraform state files' + ) + text( + name: 'TERRAFORM_CONFIG', + description: 'Terraform tfvars for the cluster_nodes module (non-airgap)', + defaultValue: '''aws_access_key = "${AWS_ACCESS_KEY_ID}" +aws_secret_key = "${AWS_SECRET_ACCESS_KEY}" +aws_ami = "ami-0600e22b69546c15c" +instance_type = "t3a.2xlarge" +aws_vpc = "vpc-bfccf4d7" +aws_subnet = "subnet-ee8cac86" +ephemeral_sg_ingress_cidrs = ["${RUNNER_PUBLIC_IP}/32"] +airgap_setup = false +proxy_setup = false +aws_volume_size = 500 +aws_volume_type = "gp3" +aws_hostname_prefix = "${HOSTNAME_PREFIX}" +aws_region = "us-east-2" +aws_route53_zone = "qa.rancher.space" +aws_ssh_user = "ubuntu" +public_ssh_key = "ssh_public_key.pub" +private_ssh_key = "~/.ssh/${AWS_SSH_PEM_KEY_NAME}" +nodes = [ + { + count = 3 + role = ["etcd", "cp", "worker"] + }, + { + count = 1 + role = ["worker"] + } +]''' + ) + text( + name: 'ANSIBLE_VARIABLES', + description: 'Ansible group_vars for RKE2 install + Rancher Helm deploy (non-airgap). Rancher version keys resolve from the pre-upgrade (FROM) version; rancher_version_upgrade keys hold the resolved target the server is upgraded to via make rancher-upgrade.', + defaultValue: '''--- +kubernetes_version: ${RKE2_VERSION} +cni: calico +kubeconfig_file: "/workspace/${INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml" +cert_manager_version: "1.20.0" +bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +password: "${RANCHER_ADMIN_PASSWORD}" +rancher_version: ${RANCHER_VERSION} +rancher_image_tag: ${RANCHER_IMAGE_TAG} +rancher_chart_repo_url: "${RANCHER_CHART_REPO_URL}" +upgrade_mode: false +rancher_version_upgrade: "${RANCHER_VERSION_UPGRADE}" +rancher_image_tag_upgrade: "${RANCHER_IMAGE_TAG_UPGRADE}" +rancher_chart_repo_upgrade: "${RANCHER_UPGRADE_REPO_NAME}" +k8s_upgrade_mode: false +server_flags: | + ingress-controller: traefik''' + ) + + // ── Test parameters ──────────────────────────────────── + string( + name: 'TEST_MODULE', + defaultValue: params.TEST_MODULE ?: 'longhorn', + description: 'rancherupgrade module under test (longhorn | neuvector | fleet | observability | all). Selects the per-phase -run regexes; fleet has no chart-upgrade phase; all runs every module suite in one pipeline.' + ) + string( + name: 'STACKSTATE_URL', + defaultValue: params.STACKSTATE_URL ?: 'http://stackstate-router.suse-observability.svc.cluster.local:8080', + description: 'observability module only: the URL the StackState agent reports to. The in-cluster router service default works when the agent is co-located with the server (the pipeline pins the server to local); override with a resolvable StackState address otherwise.' + ) + text( + name: 'CATTLE_TEST_CONFIG', + description: 'Test configuration YAML (cattle-config). rancher.adminToken is injected at runtime (initial + rotated after the server upgrade); rancherUpgradeInput.targetVersion is substituted from the resolved upgrade target.', + defaultValue: '''# The downstream per-phase executor can land on a different Jenkins agent +# than this orchestrator; awsNodeTemplate carries the three keys its +# Authorize Security Group stage needs (sg name, vpc, region) so the +# runner's public IP gets authorized on the tofu-created ephemeral SG +# (tf--sg) before the phases talk to the Rancher LB. +awsNodeTemplate: + region: us-east-2 + vpcId: vpc-bfccf4d7 + securityGroup: + - tf-${HOSTNAME_PREFIX}-sg +rancher: + host: ${HOSTNAME_PREFIX}.qa.rancher.space + adminPassword: ${ADMIN_PASSWORD} + insecure: true + # Deliberately false: pre-upgrade resources (Longhorn chart, PVC, workload) + # must survive into the post-upgrade phases; the pipeline environment + # teardown is the cleanup. + cleanup: false + # The qa-infra-automation cluster_nodes RKE2 cluster is Rancher's own + # ("local") cluster; the upgrade suite installs Longhorn on it. + clusterName: local +rancherUpgradeInput: + targetVersion: "${RANCHER_VERSION_UPGRADE}" +# Marker: set only by this pipeline. The suite fails rather than skips when +# this marker is present but the target version is missing (pipeline bug). +rancherUpgradePipeline: + enabled: true''', + ) + booleanParam( + name: 'REPORT_ARTIFACTS', + defaultValue: true, + description: 'Archive token metadata, Qase run config, and failure diagnostics as build artifacts' + ) + + // ── Downstream job parameters ───────────────────────── + string( + name: 'INDIVIDUAL_JOB', + defaultValue: params.INDIVIDUAL_JOB ?: 'go-pit-validation-job', + description: 'Name of the downstream Jenkins job that runs individual Go test suites' + ) + string( + name: 'TEST_PACKAGE', + defaultValue: './validation/rancherupgrade/...', + description: 'Go test package path passed to the downstream job' + ) + string( + name: 'TEST_TIMEOUT', + defaultValue: '60m', + description: 'Timeout for downstream test jobs' + ) + string( + name: 'QASE_REPORTER_SCRIPT', + defaultValue: 'build_qase_reporter_v2.sh', + description: 'Qase reporter script name in pipeline/scripts/' + ) + string( + name: 'QASE_SCHEMA_PREFIX', + defaultValue: 'pit', + description: 'Schema file prefix for Qase operations, for example hostbusters or pit' + ) + } + + environment { + WORKSPACE_NAME = '' + INFRA_CLEANED = 'false' + YQ_VERSION = 'v4.47.2' + // Folder-injected TEST_RUN_NAME (if present) wins; the fallback keeps + // the shared Qase run name readable when it is not injected. + TEST_RUN_NAME = "${env.TEST_RUN_NAME ?: 'Rancher Server Upgrade Validation'}" + } + + stages { + stage('Checkout') { + steps { + script { + def dirs = airgap.standardCheckout( + testsRepo: [url: params.TESTS_REPO_URL, branch: params.TESTS_BRANCH], + infraRepo: [url: params.QA_INFRA_REPO_URL, branch: params.QA_INFRA_BRANCH] + ) + env.TESTS_DIR = dirs.testsDir + env.INFRA_DIR = dirs.infraDir + + // Resolve the selected module's phase selectors before any + // infra spend: an unknown TEST_MODULE fails here, and the + // log records exactly which regex each stage will run. + TEST_PHASES = UPGRADE_TEST_MODULES[params.TEST_MODULE] + if (!TEST_PHASES) { + error "Unknown TEST_MODULE '${params.TEST_MODULE}' — valid modules: ${UPGRADE_TEST_MODULES.keySet().sort().join(', ')}" + } + echo "TEST_MODULE '${params.TEST_MODULE}' phase selectors: pre=${TEST_PHASES.pre} post=${TEST_PHASES.post} chart=${TEST_PHASES.chart ?: '(none: module has no chart-upgrade phase)'}" + + def hostnamePrefix = params.HOSTNAME_PREFIX ?: 'unknown' + currentBuild.displayName = "#${env.BUILD_NUMBER} - ${hostnamePrefix}" + currentBuild.description = "Rancher server upgrade validation for ${hostnamePrefix}" + } + } + } + + stage('Build Docker Images') { + steps { + // rancher-infra-tools — used by make.groovy / infra.* for all make + // targets (library resolves the :latest alias); direct invocations + // in this file use the build-pinned tag. + sh "docker build --no-cache --platform linux/amd64 -t ${INFRA_TOOLS_IMAGE} -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." + // rancher-go-test — used for the qase binary build, token playbook, + // and redaction step, always via the build-pinned tag. + sh "docker build --no-cache --platform linux/amd64 -t ${GO_TEST_IMAGE} -t rancher-go-test:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.airgap-go-tests ." + } + } + + stage('Setup Tools') { + steps { + sh """ + wget -qO ./yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + wget -qO ./yq_checksums https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/checksums + grep -q '^yq_linux_amd64 ' ./yq_checksums + awk '/^yq_linux_amd64 / {print \$19 \" ./yq\"}' ./yq_checksums | sha256sum -c - + rm -f ./yq_checksums + chmod +x ./yq + """ + } + } + + // Auto-detect this runner's public IP so the ephemeral security + // group's SSH/LB ingress rule (ephemeral_sg_ingress_cidrs) allows + // the IP that Ansible/SSH will actually originate from. + stage('Detect Public IP') { + steps { + script { + env.RUNNER_PUBLIC_IP = infrastructure.detectPublicIp() + } + } + } + + // ── Resolve the upgrade window ───────────────────────── + // Empty params resolve at runtime via qa-infra-automation's + // rancher-resolve-version target; explicit pins pass through. + // The chosen lines/regexes and resolved versions are echoed so CI + // logs show exactly which version each stage ran against. + stage('Resolve Rancher Versions') { + steps { + script { + env.RANCHER_FROM_VERSION = params.RANCHER_VERSION_FROM?.trim() ?: '' + env.RANCHER_TO_VERSION = params.RANCHER_VERSION_TO_UPGRADE?.trim() ?: '' + + if (!env.RANCHER_FROM_VERSION) { + echo 'RANCHER_VERSION_FROM empty — resolving latest 2.14.x patch (SOURCE_LINE=2.14, latest final of the line)' + def resolveOut = make.runTarget( + target: 'rancher-resolve-version', + dir: env.INFRA_DIR, + makeArgs: [ + 'SOURCE_LINE=2.14', + "RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}" + ], + returnStdout: true, + passAwsCreds: false + ) + writeFile file: 'resolve-from.json', text: resolveOut + env.RANCHER_FROM_VERSION = sh( + script: "./yq '.chart_version' resolve-from.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + env.RANCHER_FROM_IMAGE_TAG = sh( + script: "./yq '.image_tag' resolve-from.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + } else { + env.RANCHER_FROM_IMAGE_TAG = env.RANCHER_FROM_VERSION.startsWith('v') ? env.RANCHER_FROM_VERSION : "v${env.RANCHER_FROM_VERSION}" + echo "RANCHER_VERSION_FROM pinned: ${env.RANCHER_FROM_VERSION} (image tag ${env.RANCHER_FROM_IMAGE_TAG})" + } + + if (!env.RANCHER_TO_VERSION) { + echo 'RANCHER_VERSION_TO_UPGRADE empty — resolving latest released final (no SOURCE_LINE, repo index latest final)' + def resolveOut = make.runTarget( + target: 'rancher-resolve-version', + dir: env.INFRA_DIR, + makeArgs: ["RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}"], + returnStdout: true, + passAwsCreds: false + ) + writeFile file: 'resolve-to.json', text: resolveOut + env.RANCHER_TO_VERSION = sh( + script: "./yq '.chart_version' resolve-to.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + env.RANCHER_TO_IMAGE_TAG = sh( + script: "./yq '.image_tag' resolve-to.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + } else { + env.RANCHER_TO_IMAGE_TAG = env.RANCHER_TO_VERSION.startsWith('v') ? env.RANCHER_TO_VERSION : "v${env.RANCHER_TO_VERSION}" + echo "RANCHER_VERSION_TO_UPGRADE pinned: ${env.RANCHER_TO_VERSION} (image tag ${env.RANCHER_TO_IMAGE_TAG})" + } + + echo """ + Upgrade window resolved: + FROM (deploy): ${env.RANCHER_FROM_VERSION} (${env.RANCHER_FROM_IMAGE_TAG}) + TO (upgrade): ${env.RANCHER_TO_VERSION} (${env.RANCHER_TO_IMAGE_TAG}) + Chart flavor: ${params.RANCHER_CHART_REPO_FLAVOR} + """.stripIndent() + currentBuild.description = "Upgrade ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (${params.RANCHER_CHART_REPO_FLAVOR}) | ${params.HOSTNAME_PREFIX} | ${params.TEST_MODULE}" + + if (params.TEST_MODULE in ['observability', 'all']) { + // Pre-deploy guard: fail before any infra spend when + // the SUSE Observability license credential is absent + // or empty; the post-deploy config injection reuses it. + def licenseProblem = '' + try { + withCredentials([string(credentialsId: 'STACKSTATE_LICENSE', variable: 'STACKSTATE_LICENSE_CHECK')]) { + if (!env.STACKSTATE_LICENSE_CHECK?.trim()) { + licenseProblem = 'STACKSTATE_LICENSE exists but is empty; set the SUSE Observability license key and rerun' + } + } + } catch (Exception e) { + licenseProblem = "observability module needs the STACKSTATE_LICENSE folder credential (SUSE Observability license key); add it to the rancher_qa folder and rerun (checked pre-deploy to avoid infra spend)" + } + if (licenseProblem) { + error licenseProblem + } + } + } + } + } + + stage('Deploy Rancher Pre-Upgrade Environment') { + steps { + script { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/default" + def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes" + def workspaceName = '' + + // ── SSH key setup ────────────────────────── + stage('Configure SSH Key') { + infrastructure.writeSshKey( + keyContent: env.AWS_SSH_PEM_KEY, + keyName: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ) + // Copy public key into the tofu module directory so + // tofu's file() function can read it (restricted to + // the module directory tree). + sh "cp .ssh/*.pub ${tofuModulePath}/ssh_public_key.pub" + } + + // ── Render terraform.tfvars ───────────────── + // Must exist before any make target runs (check-tofu-dir validates it). + stage('Configure Tofu Variables') { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME, + 'RUNNER_PUBLIC_IP': env.RUNNER_PUBLIC_IP + ] + ) + infrastructure.writeConfig( + path: "${tofuModulePath}/terraform.tfvars", + content: terraformConfig + ) + } + + // ── Initialize S3 backend ─────────────────── + stage('Initialize Tofu Backend') { + make.runTarget( + target: 'backend-s3', + dir: env.INFRA_DIR, + makeArgs: [ + "BUCKET=${params.S3_BUCKET_NAME}", + "KEY=${params.S3_KEY_PREFIX}", + "REGION=${params.S3_BUCKET_REGION}" + ] + ) + } + + // ── Create tofu workspace ─────────────────── + stage('Create Workspace') { + workspaceName = infrastructure.generateWorkspaceName( + prefix: 'jenkins_rancher_upgrade_workspace', + suffix: params.HOSTNAME_PREFIX, + includeTimestamp: false + ) + make.runTarget( + target: 'workspace-new', + dir: env.INFRA_DIR, + makeArgs: "WORKSPACE=${workspaceName}" + ) + infrastructure.archiveWorkspaceName(workspaceName: workspaceName) + env.WORKSPACE_NAME = workspaceName + } + + // ── Provision infra + generate inventory ──── + stage('Provision Infrastructure') { + make.runTarget( + target: 'infra-up', + dir: env.INFRA_DIR, + makeArgs: "WORKSPACE=${workspaceName}" + ) + } + + // ── Write group_vars/all.yml ──────────────── + // Must exist before make cluster (make infra-up generates + // inventory.yml but not the group_vars subdirectory). + // Rancher version keys carry the resolved FROM version; + // the *_upgrade keys carry the resolved TO version the + // later `make rancher-upgrade` stage targets. + stage('Configure Ansible Variables') { + sh "mkdir -p ${ansiblePath}/inventory/group_vars" + + // Flavor selects the deploy-side repo too: the prime + // leg installs the FROM version from the prime index + // (community stops the 2.14 line at 2.14.3 while prime + // continues it), keeping resolve, deploy, and upgrade + // all on the same artifact line. + def fromChartRepoUrl = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'https://charts.rancher.com/server-charts/prime' : 'https://releases.rancher.com/server-charts/latest' + def upgradeRepoName = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'rancher-prime-upgrade' : 'rancher-upgrade' + + def ansibleVars = infrastructure.parseAndSubstituteVars( + content: params.ANSIBLE_VARIABLES, + envVars: [ + 'RKE2_VERSION': params.RKE2_VERSION, + 'INFRA_DIR': env.INFRA_DIR, + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_VERSION': env.RANCHER_FROM_VERSION, + 'RANCHER_IMAGE_TAG': env.RANCHER_FROM_IMAGE_TAG, + 'RANCHER_CHART_REPO_URL': fromChartRepoUrl, + 'RANCHER_VERSION_UPGRADE': env.RANCHER_TO_VERSION, + 'RANCHER_IMAGE_TAG_UPGRADE': env.RANCHER_TO_IMAGE_TAG, + 'RANCHER_UPGRADE_REPO_NAME': upgradeRepoName + ] + ) + + def rancherVars = infrastructure.parseAndSubstituteVars( + content: '''deploy_rancher: true +install_helm: true +rancher_hostname: "${HOSTNAME_PREFIX}.qa.rancher.space" +rancher_bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +rancher_admin_password: "${RANCHER_ADMIN_PASSWORD}" +rancher_use_bundled_system_charts: true +''', + envVars: [ + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD + ] + ) + + // Merge ANSIBLE_VARIABLES + Rancher vars into a single group_vars/all.yml + infrastructure.writeConfig( + path: "${ansiblePath}/inventory/group_vars/all.yml", + content: ansibleVars + '\n' + rancherVars + ) + } + + // ── Deploy RKE2 cluster (ansible; no AWS creds) ── + stage('Deploy RKE2 Cluster') { + make.runTarget( + target: 'cluster', + dir: env.INFRA_DIR, + passAwsCreds: false + ) + } + + // ── Deploy pre-upgrade Rancher (helm; no AWS creds) ── + stage('Deploy Rancher') { + make.runTarget( + target: 'rancher', + dir: env.INFRA_DIR, + passAwsCreds: false + ) + } + + // ── Capture Rancher hostname ──────────────── + stage('Output Infrastructure Details') { + // Rancher hostname is deterministically constructed from HOSTNAME_PREFIX + // matching the value set in rancherVars (rancher_hostname). + def rancherHostname = "${params.HOSTNAME_PREFIX}.qa.rancher.space" + echo "Rancher Hostname: https://${rancherHostname}" + env.RANCHER_HOSTNAME = rancherHostname + } + } + } + } + } + + stage('Create Test Config and Inject Admin Token') { + steps { + script { + def workspace = pwd() + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + + // Create cattle-config with variable substitution, including + // the resolved upgrade target for rancherUpgradeInput.targetVersion. + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY' + ]) { + def resolvedConfig = infrastructure.parseAndSubstituteVars( + content: params.CATTLE_TEST_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_VERSION_UPGRADE': env.RANCHER_TO_VERSION + ] + ) + writeFile file: cattleConfigPath, text: resolvedConfig.trim() + '\n' + } + + // The fleet module's pre-upgrade phase serves its GitRepo + // over SSH from a downstream node. Downstream executors + // mount no .ssh, so the node key is threaded through the + // test config for fleet alone; the PEM sits in the + // orchestrator workspace where writeSshKey put it for + // ansible. + if (params.TEST_MODULE in ['fleet', 'all']) { + property.useWithProperties(['AWS_SSH_PEM_KEY_NAME']) { + def pemPath = "${workspace}/.ssh/${AWS_SSH_PEM_KEY_NAME}" + if (!fileExists(pemPath)) { + error "fleet module needs the node key at ${pemPath} (AWS_SSH_PEM_KEY_NAME), but it does not exist" + } + def sshUser = sh( + script: "awk -F': *' '/^aws_ssh_user/{print \$2}' ${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes/terraform.tfvars | head -1 | tr -d '\"'", + returnStdout: true + ).trim() + if (!sshUser) { + sshUser = 'ubuntu' + } + sh """ + set +x + export nodekey=\$(cat ${pemPath}) + export sshuser='${sshUser}' + ./yq e '.sshNode.sshKey = strenv(nodekey)' -i ${cattleConfigPath} + ./yq e '.sshNode.sshUser = strenv(sshuser)' -i ${cattleConfigPath} + """ + echo "fleet module: node ssh key and user threaded into the test config" + } + } + + if (params.TEST_MODULE in ['observability', 'all']) { + // SUSE Observability needs its license plus the + // connection settings the server and the agent share. + // The license is the STACKSTATE_LICENSE folder + // credential, checked pre-deploy by the guard in the + // version-resolution stage; the URL defaults to the + // in-cluster router service, which the co-located + // agent can always reach; the cluster API key is + // self-consistent because this suite installs the + // server with the same value. + property.useWithProperties(['STACKSTATE_LICENSE']) { + if (!env.STACKSTATE_LICENSE) { + error "observability module needs the STACKSTATE_LICENSE folder credential (SUSE Observability license key); add it to the rancher_qa folder and rerun" + } + sh """ + set +x + export sslicense=\$STACKSTATE_LICENSE + ./yq e '.stackstateConfigs.url = "${params.STACKSTATE_URL}"' -i ${cattleConfigPath} + ./yq e '.stackstateConfigs.license = strenv(sslicense)' -i ${cattleConfigPath} + ./yq e '.stackstateConfigs.clusterApiKey = "rancher-upgrade-observability-cluster-key"' -i ${cattleConfigPath} + ./yq e '.stackstateConfigs.adminPassword = "rancher-upgrade-observability-admin"' -i ${cattleConfigPath} + """ + echo "observability module: stackstate connection settings threaded into the test config" + } + } + + // Generate admin token via Ansible playbook + // TODO: replace with 'make token' once that target is added upstream + def containerConfigPath = "/workspace/${cattleConfigPath}" + def tokenTtl = env.RANCHER_TOKEN_TTL ?: '0' + def tokenDescription = "jenkins-rancher-upgrade-${env.BUILD_NUMBER}" + + // Diagnostic logging to validate env + output path before token generation + echo "Token generation context: workspace=${workspace}, config=${containerConfigPath}" + echo "RANCHER_ADMIN_PASSWORD present in params: ${params.RANCHER_ADMIN_PASSWORD ? 'yes' : 'no'}" + + sh """ + docker run --rm --platform linux/amd64 \ + --name generate-token \ + -e RANCHER_ADMIN_PASSWORD \ + -e RANCHER_URL=https://${params.HOSTNAME_PREFIX}.qa.rancher.space \ + -e ANSIBLE_CONFIG=/workspace/${env.INFRA_DIR}/ansible/ansible.cfg \ + -v ${workspace}:/workspace \ + -w /workspace/${env.INFRA_DIR}/ansible/rke2/default \ + ${GO_TEST_IMAGE} \ + ansible-playbook -i inventory/inventory.yml /workspace/${env.INFRA_DIR}/ansible/rancher/token/generate-admin-token.yml \ + -e rancher_cattle_config_file=${containerConfigPath} \ + -e rancher_token_ttl=${tokenTtl} \ + -e rancher_token_description=${tokenDescription} \ + -e rancher_token_output_format=json \ + -e rancher_token_output_file=/workspace/rancher-token.json \ + -e rancher_url=https://${params.HOSTNAME_PREFIX}.qa.rancher.space + """ + + def tokenFileExists = fileExists('rancher-token.json') + echo "Token file present after generation: ${tokenFileExists}" + + if (tokenFileExists) { + // Redact secret fields from token JSON via Docker container. + // Pipeline Utility Steps plugin (readJSON/writeJSON) is not installed, + // and python3 is not available on the Jenkins agent host. + sh "docker run --rm -v \$(pwd):/workspace -w /workspace ${GO_TEST_IMAGE} python3 -c \"import json; d=json.load(open('rancher-token.json')); [d.pop(k,None) for k in ('token','bearerToken','accessKey','secretKey','value')]; json.dump(d,open('rancher-token-metadata.json','w'),indent=2)\"" + if (params.REPORT_ARTIFACTS) { + archiveArtifacts artifacts: 'rancher-token-metadata.json' + echo 'Archived redacted token metadata: rancher-token-metadata.json' + } + + // Infrastructure, Rancher deploy, and admin token all succeeded. + // Shared library helpers (infra.*) may have set currentBuild.result to + // UNSTABLE/FAILURE via internal catchError() wrappers for non-fatal warnings, + // which would cause declarative pipeline to skip downstream test stages. + // Reset build result to SUCCESS so the upgrade phases can execute. + if (currentBuild.result && currentBuild.result != 'SUCCESS') { + echo "Resetting currentBuild.result from '${currentBuild.result}' to 'SUCCESS' — token generation succeeded, allowing upgrade phases to run" + currentBuild.result = 'SUCCESS' + } + } else { + error 'rancher-token.json not found after token generation — cannot run test phases without an admin token' + } + } + } + } + + // ── Phase 1: pre-upgrade module install + state seed ── + // Starts the shared Qase run once; every phase reports into it. + stage('Pre-Upgrade Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def workspace = pwd() + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + def testRunConfig = "${workspace}/testrunconfig.yaml" + def qaseBinary = "${workspace}/testrun" + + // Build qase testrun binary into the workspace root so the + // agent-side invocation below and testrunconfig.yaml (written + // to the binary's CWD) line up. + sh """ + docker run --rm --platform linux/amd64 \ + -v ${workspace}:/workspace \ + -w /workspace/${env.TESTS_DIR} \ + ${GO_TEST_IMAGE} \ + sh -c 'env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /workspace/testrun ./validation/pipeline/qase/testrun' + """ + + // Start the shared qase test run (completed after the final phase) + sh "${qaseBinary} --startRun=true" + + // Read test run ID + def testRunID = '' + if (fileExists(testRunConfig)) { + testRunID = sh( + script: "./yq '.id' ${testRunConfig}", + returnStdout: true + ).trim() + } + echo "Shared Qase test run started: id=${testRunID}" + if (params.REPORT_ARTIFACTS && fileExists(testRunConfig)) { + archiveArtifacts artifacts: 'testrunconfig.yaml', allowEmptyArchive: true + } + + // NOTE: unlike Jenkinsfile.e2e, rancher.cleanup is NOT set to + // true and no desiredflags are forced here — the upgrade suite's + // cross-phase state must survive, and the pipeline environment + // teardown is the cleanup. + def configYaml = readFile(file: cattleConfigPath) + + echo "Dispatching pre-upgrade phase: GOTEST_TESTCASE ${TEST_PHASES.pre}" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: TEST_PHASES.pre), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } + } + } + } + + // ── Phase 2: in-place Rancher server upgrade ── + // `make rancher-upgrade` rotates the Rancher admin token and writes the + // new one to ansible/rancher/default-ha/generated.tfvars; the old token + // is revoked, so the test config MUST be re-injected before the + // post-upgrade phases dispatch (verified live: old token 401s). + stage('Upgrade Rancher Server') { + steps { + script { + echo "Upgrading Rancher server: ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (flavor ${params.RANCHER_CHART_REPO_FLAVOR})" + make.runTarget( + target: 'rancher-upgrade', + dir: env.INFRA_DIR, + makeArgs: [ + "ENV=default", + "RANCHER_VERSION_TO_UPGRADE=${env.RANCHER_TO_VERSION}", + "RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}" + ], + passAwsCreds: false + ) + + // Re-inject the rotated admin token into the test config + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + def generatedTfvars = "${env.INFRA_DIR}/ansible/rancher/default-ha/generated.tfvars" + + if (!fileExists(generatedTfvars)) { + error "Rotated token file not found at ${generatedTfvars} — post-upgrade phases cannot authenticate" + } + def rotatedToken = sh( + script: "awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}", + returnStdout: true + ).trim() + if (!rotatedToken) { + error "Could not extract api_key from ${generatedTfvars} — post-upgrade phases cannot authenticate" + } + echo "Rotated admin token extracted (length ${rotatedToken.length()}); re-injecting into test config" + + // The token never leaves this shell step (no Groovy/env + // interpolation, set +x keeps it out of the xtrace), + // and the injected token is smoke-checked against the + // live API before any phase is dispatched on it. + sh """ + set +x + token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) + # export: yq's strenv() reads the environment, not the + # shell's unexported variables (empty-token 401 root cause). + export token + ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} + status=\$(curl -sk -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) + if [ "\$status" != "200" ]; then + echo "Rotated token rejected by the upgraded Rancher API: HTTP \$status (token length \${#token})" + exit 1 + fi + echo "Rotated token smoke-check: HTTP 200 from the upgraded Rancher API" + """ + echo "Test config now carries the post-upgrade admin token; rancherUpgradeInput.targetVersion=${env.RANCHER_TO_VERSION}" + } + } + } + + // ── Phase 3: post-upgrade assertions ── + stage('Post-Upgrade Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def testRunID = sh( + script: "./yq '.id' testrunconfig.yaml", + returnStdout: true + ).trim() + def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + + // Fail fast (without printing the token) if the config + // about to be dispatched does not carry the rotated one. + sh """ + set +x + expected=\$(awk -F'\"' '/^api_key/{print \$2}' ${env.INFRA_DIR}/ansible/rancher/default-ha/generated.tfvars) + actual=\$(./yq '.rancher.adminToken' ${env.TESTS_DIR}/cattle-config.yaml) + if [ "\$expected" != "\$actual" ]; then + echo "Dispatched config adminToken does not match the rotated token (expected length \${#expected}, got length \${#actual})" + exit 1 + fi + echo "Dispatched config carries the rotated admin token (length \${#expected})" + """ + + echo "Dispatching post-upgrade phase: GOTEST_TESTCASE ${TEST_PHASES.post} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: TEST_PHASES.post), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + + // Modules without a chart-upgrade phase (fleet) end + // their pipeline here: close the shared Qase run now. + // The post{always} fallback below still covers every + // failure path for every module. + if (!TEST_PHASES.chart) { + sh "./testrun --startRun=false" + env.QASE_RUN_COMPLETED = 'true' + echo "Shared Qase test run ${testRunID} completed (module has no chart-upgrade phase)" + } + } + } + } + } + + // ── Phase 4: module chart upgrade through the upgraded catalog ── + // Completes the shared Qase run after this final phase; do not rely on + // the downstream job's own QASE_TEST_RUN_COMPLETE for the shared run. + // Skipped entirely for modules without a chart phase (fleet): their + // post-upgrade stage closes the shared Qase run instead. + stage('Chart-Upgrade Tests') { + when { + expression { TEST_PHASES?.chart } + } + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def testRunID = sh( + script: "./yq '.id' testrunconfig.yaml", + returnStdout: true + ).trim() + def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + + // All-mode sequencing: the volume-heavy observability + // chart upgrade runs BEFORE longhorn's. Longhorn's + // upgrade live-upgrades every volume engine in the + // cluster, and an observability rollout starting while + // those engine upgrades are in flight can stall its + // statefulsets on volume reattach for the full helm + // deadline (build 52: zookeeper stuck in + // ContainerCreating for 45 minutes, kafka crashlooping + // on the unresolved headless service, everything + // cascading from there). + def chartSelectors = params.TEST_MODULE == 'all' ? [ + UPGRADE_TEST_MODULES.observability.chart, + '-run "Test(Fleet|Longhorn|NeuVector)UpgradeTestSuite/Test(Fleet|Longhorn|NeuVector)ChartUpgrade"' + ] : [TEST_PHASES.chart] + + chartSelectors.each { selector -> + echo "Dispatching chart-upgrade phase: GOTEST_TESTCASE ${selector} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: selector), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } + + // Complete the shared qase test run + sh "./testrun --startRun=false" + env.QASE_RUN_COMPLETED = 'true' + echo "Shared Qase test run ${testRunID} completed" + } + } + } + } + } + + post { + always { + script { + // Failure diagnostics run first, while the live cluster and + // the docker images still exist: Jenkins evaluates post + // conditions in a fixed order (always before failure, cleanup + // last), so every failure-path action must run in this block, + // ahead of the teardown below. + if (currentBuild.currentResult != 'SUCCESS') { + collectUpgradeDiagnostics() + } + + // Complete the shared Qase run when a phase failed before the + // final stage could close it (no-op once already completed). + if (env.QASE_RUN_COMPLETED != 'true' && fileExists('testrun') && fileExists('testrunconfig.yaml')) { + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + sh './testrun --startRun=false' + } + env.QASE_RUN_COMPLETED = 'true' + } + } + + // Single teardown decision point. The failure-path destroy is + // decided here rather than in a failure{} block because always + // runs first: destroyInfrastructure needs workspace_name.txt, + // the tfvars and the infra-tools image, which are only removed + // afterwards in cleanup{}. A failed run is governed by + // DESTROY_ON_FAILURE alone and a successful one by + // DESTROY_AFTER_TESTS, so DESTROY_ON_FAILURE=false on a failed + // run keeps the environment around for debugging even when + // DESTROY_AFTER_TESTS is true. + def buildSucceeded = currentBuild.currentResult == 'SUCCESS' + def shouldDestroy = buildSucceeded ? params.DESTROY_AFTER_TESTS : params.DESTROY_ON_FAILURE + if (shouldDestroy) { + destroyInfrastructure(buildSucceeded ? 'DESTROY_AFTER_TESTS' : 'DESTROY_ON_FAILURE') + } + + } + } + + cleanup { + script { + // Docker cleanup — cleanup{} runs after always, so the + // teardown above can still resolve the infra-tools image. + sh "docker rmi -f ${GO_TEST_IMAGE} ${INFRA_TOOLS_IMAGE} rancher-go-test:latest rancher-infra-tools:latest || true" + + // Sensitive artifact cleanup — shred or remove credentials from workspace + sh ''' + set +e + + cleanup_file() { + if [ -f "$1" ]; then + if command -v shred >/dev/null 2>&1; then + shred -u "$1" || rm -f "$1" + else + rm -f "$1" + fi + fi + } + + cleanup_dir() { + if [ -d "$1" ]; then + find "$1" -type f -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + rm -rf "$1" + fi + } + + cleanup_file ".go_test_env" + cleanup_file "rancher-token.json" + cleanup_file "rancher-token-metadata.json" + cleanup_file "workspace_name.txt" + cleanup_file "resolve-from.json" + cleanup_file "resolve-to.json" + # cattle-config carries the admin password and the live rotated + # admin token; group_vars/all.yml carries the rancher passwords. + cleanup_file "${TESTS_DIR}/cattle-config.yaml" + + if [ -n "${INFRA_DIR}" ] && [ -d "${INFRA_DIR}" ]; then + find "${INFRA_DIR}" -type f \\( -name 'terraform.tfvars' -o -name 'generated.tfvars' -o -name 'all.yml' -o -name '*.pem' -o -name 'id_rsa' -o -name 'id_ed25519' \\) -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + fi + + cleanup_dir ".ssh" + ''' + } + } + } +} diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md new file mode 100644 index 0000000000..901b0f5b32 --- /dev/null +++ b/validation/rancherupgrade/README.md @@ -0,0 +1,101 @@ +# Rancher Upgrade + +The `rancherupgrade` package validates upgrading the Rancher server from the previous +release line to the latest (currently 2.14.x -> 2.15.x) with interoperability products +installed. The line pair is the pipeline's default, not a suite constraint: the suite +itself is version-agnostic and gates on `rancherUpgradeInput.targetVersion`, while the +pipeline resolves the source via `RANCHER_VERSION_FROM` (empty = latest 2.14.x patch via +`SOURCE_LINE=2.14`) and the target via `RANCHER_VERSION_TO_UPGRADE` (empty = latest +released final). The first vertical is Longhorn: a pre-upgrade +phase installs the Longhorn chart and seeds a checksum workload on a Longhorn volume, +the Rancher server is then upgraded out-of-band (qa-infra-automation `make rancher-upgrade`), +and post-upgrade phases verify the server version, the surviving Longhorn installation, +the workload data, and a Longhorn chart upgrade to the version offered by the upgraded +Rancher's catalog. + +Each phase is a suite test method selected via the `-run` regex and runs as its own +`go test` invocation in its own pipeline stage. All cross-phase state lives in-cluster +under deterministic names (PVC/deployment `rancher-upgrade-longhorn`, data file +`/auto-mnt/checksum-data`), never in process memory. + +## Deliberate deviations + +Do not "fix" the following; both are required by the upgrade workflow (rancher/tests#967): + +- `TearDownSuite` does not call `session.Cleanup()` and the helpers register no cleanup + funcs: pre-phase resources (Longhorn chart, PVC, workload) must survive into the + later phases, which run as separate processes. The pipeline environment teardown is + the cleanup. +- The suite is admin-only: upgrade phases assert server-level state and the pipeline + runs them with the admin token. +- The server upgrade stays out-of-band (a pipeline stage running qa-infra-automation + `make rancher-upgrade`) rather than a step inside the test: the go test phases + assert outcomes only and hold no infra credentials, the upgrade needs the + controller-side qa-infra checkout and its terraform/ansible credentials, and + per-phase Jenkins stage status is the acceptance signal; pulling the upgrade into + the test process would collapse the three stages into one. +- The build tag is `(validation || pit.weekly)`: generic scheduled PIT legs deploy + Rancher once and never upgrade it, so `SetupSuite` skips the suite when + `rancherUpgradeInput.targetVersion` is unset. The dedicated pipeline marks its + config with `rancherUpgradePipeline.enabled: true`; with the marker present but + the target missing, the suite fails (pipeline bug) instead of skipping. + +## Config + +```yaml +rancher: + host: + adminToken: + insecure: true + clusterName: "local" # Rancher's own local cluster: the qa-infra-automation RKE2 HA deploy + +rancherUpgradeInput: + targetVersion: "" +``` +`rancher.clusterName` selects the target cluster. The pipeline uses `local` — Rancher's own +3-node HA RKE2 cluster provisioned by qa-infra-automation — so Longhorn and its volume are +exercised on the server cluster itself, matching what the pipeline deploys. +`rancherUpgradeInput.targetVersion` is required by every phase (the suite skips without it); +the pipeline writes it from the qa-infra-automation resolver output, or set it by hand for +local runs. + +## Running + +Pre-upgrade (installs Longhorn at the 2.14-era latest and seeds the checksum workload; +run against the not-yet-upgraded Rancher): + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade" +``` + +Post-upgrade (after `make rancher-upgrade RANCHER_VERSION_TO_UPGRADE= RANCHER_CHART_REPO_FLAVOR=community|prime` +in qa-infra-automation, and with `rancherUpgradeInput.targetVersion` set): + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade" +``` + +Chart upgrade: + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade" +``` + +## Notes + +- The target cluster is Rancher's own local cluster (the qa-infra-automation 3-node HA + RKE2 deploy), so the server nodes themselves must be sized for Longhorn plus the later + NeuVector/Observability suites; this is an infra concern and is not configurable here. +- The Rancher server upgrade itself is `make rancher-upgrade` in + [qa-infra-automation](https://github.com/rancher/qa-infra-automation); this package + only runs the pre- and post-upgrade assertions around it. +- `make rancher-upgrade` rotates the admin token and writes the new one to + `ansible/rancher/-ha/generated.tfvars`; the pipeline must copy it into the test + config's `rancher.adminToken` before the post-upgrade phases (the pre-upgrade token + is revoked by the upgrade). +- After a server upgrade the cluster-scoped catalog follower can keep serving the + previous Rancher line's index (e.g. release-v2.14 after upgrading to 2.15) until the + `cattle-cluster-agent` re-syncs. The chart-upgrade phase detects this and fails with + an explicit stale-catalog message when the catalog's latest equals the installed + version, or with "failed to find chart longhorn from the chart repo" on a lookup + miss; restart the cluster agent and re-run the phase. Observed re-sync time ~4 minutes. diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go new file mode 100644 index 0000000000..3b203420d6 --- /dev/null +++ b/validation/rancherupgrade/longhorn_test.go @@ -0,0 +1,182 @@ +//go:build (validation || pit.weekly) && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 + +package rancherupgrade + +import ( + "testing" + + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + "github.com/rancher/tests/actions/charts" + actionsClusters "github.com/rancher/tests/actions/clusters" + "github.com/rancher/tests/actions/projects" + "github.com/rancher/tests/actions/storage" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type LonghornUpgradeTestSuite struct { + suite.Suite + session *session.Session + client *rancher.Client + cluster *clusters.ClusterMeta +} + +const ( + // checksumWorkloadName is the deterministic name shared by the PVC and the nginx + // deployment created in the pre-upgrade phase and relocated by later phases. + checksumWorkloadName = "rancher-upgrade-longhorn" + + // checksumContent is the shell-safe data written to the checksum file on the + // Longhorn-backed volume before the Rancher upgrade and verified after it. + checksumContent = "rancher-upgrade-longhorn-checksum-v1" +) + +// loadUpgradeInput returns the rancherUpgradeInput section of the test config. +func loadUpgradeInput() *UpgradeInput { + upgradeInput := new(UpgradeInput) + config.LoadConfig(UpgradeInputConfigKey, upgradeInput) + + return upgradeInput +} + +func (s *LonghornUpgradeTestSuite) SetupSuite() { + cfg := loadUpgradeInput() + pipeline := new(PipelineInput) + config.LoadConfig(PipelineConfigKey, pipeline) + + if cfg.TargetVersion == "" { + // The dedicated pipeline always writes both keys, so an empty target + // there is a pipeline bug and must fail, not skip. + require.False(s.T(), pipeline.Enabled, "rancher-upgrade pipeline config is missing %s.targetVersion", UpgradeInputConfigKey) + // Generic PIT legs never upgrade the server between phases; without + // an upgrade target the phases have nothing to assert, so skip. + s.T().Skipf("%s.targetVersion is not set: this suite runs only in the rancher-upgrade pipeline, which upgrades the server between phases", UpgradeInputConfigKey) + } + + testSession := session.NewSession() + s.session = testSession + + client, err := rancher.NewClient("", testSession) + require.NoError(s.T(), err) + s.client = client + + s.cluster, err = clusters.NewClusterMeta(client, client.RancherConfig.ClusterName) + require.NoError(s.T(), err) +} + +// TearDownSuite deliberately does not call session.Cleanup(): resources created in +// the pre-upgrade phase (Longhorn chart, PVC, workload) must survive into the +// post-upgrade and chart-upgrade phases, which run as separate go test processes. +// The pipeline environment teardown is the cleanup for these resources. +func (s *LonghornUpgradeTestSuite) TearDownSuite() { +} + +func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { + logrus.Infof("pre-upgrade phase: creating project on cluster %s", s.cluster.ID) + project, _, err := projects.CreateProjectAndNamespace(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + // Resolve the version from the cluster-scoped catalog and install that: the + // management view can offer versions the cluster-scoped shard does not serve + // yet (rancher-2.15-catalog-sharding-issue.md), and the chart action validates + // against the cluster-scoped index. The pre-phase still yields the 2.14-era + // version by construction, since it runs against the pre-upgrade Rancher. + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + version, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: installing longhorn %s from the %s repo", version, catalog.RancherChartRepo) + + err = charts.InstallLonghornChart(s.client, charts.PayloadOpts{ + Namespace: charts.LonghornNamespace, + Host: s.client.RancherConfig.Host, + InstallOptions: charts.InstallOptions{ + Cluster: s.cluster, + Version: version, + ProjectID: project.ID, + }, + }, nil) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: longhorn %s installed; seeding checksum workload %s", version, checksumWorkloadName) + + err = storage.CreateChecksumWorkload(s.client, s.cluster.ID, charts.LonghornStorageClass, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: checksum workload %s ready; pre-upgrade state seeded", checksumWorkloadName) +} + +func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("post-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: server version %s observed; checking longhorn survival", cfg.TargetVersion) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.True(s.T(), chart.IsAlreadyInstalled, "longhorn chart must still be installed after the Rancher server upgrade") + logrus.Infof("post-upgrade phase: longhorn chart still installed; verifying workload data") + + err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: checksum verified on %s", checksumWorkloadName) +} + +func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("chart-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + + // Resolve the latest version from the cluster-scoped catalog: the management + // view can report versions absent from the cluster-scoped index (see + // rancher-2.15-catalog-sharding-issue.md at the repo root). + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + latest, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + // Guard against a stale cluster-scoped catalog: until the cluster agent + // re-syncs after the server upgrade, the follower can still serve the + // previous Rancher line's index, whose latest equals the installed version. + // Upgrading to it would pass vacuously (a same-version upgrade asserted + // equal to itself), so fail with the runbook remedy instead; see README Notes. + installed, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.NotEqual(s.T(), + installed.ChartDetails.Spec.Chart.Metadata.Version, latest, + "cluster-scoped catalog still serves the pre-upgrade index: installed longhorn %s equals latest %s; restart the cluster agent and re-run this phase", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + logrus.Infof("chart-upgrade phase: upgrading longhorn %s to %s from the upgraded catalog", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + + installOptions := &charts.InstallOptions{ + Cluster: s.cluster, + Version: latest, + } + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return charts.UpgradeLonghornChart(s.client, installOptions) + }) + require.NoError(s.T(), err) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.Equal(s.T(), latest, chart.ChartDetails.Spec.Chart.Metadata.Version) + logrus.Infof("chart-upgrade phase: longhorn at %s; verifying workload data", latest) + + err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) + logrus.Infof("chart-upgrade phase: checksum verified on %s", checksumWorkloadName) +} + +func TestLonghornUpgradeTestSuite(t *testing.T) { + suite.Run(t, new(LonghornUpgradeTestSuite)) +} diff --git a/validation/rancherupgrade/rancherupgrade.go b/validation/rancherupgrade/rancherupgrade.go new file mode 100644 index 0000000000..5d3aeb112a --- /dev/null +++ b/validation/rancherupgrade/rancherupgrade.go @@ -0,0 +1,29 @@ +// Package rancherupgrade validates the Rancher server upgrade path (2.14.x -> latest) +// with interoperability products installed. Each phase (pre-upgrade, post-upgrade, +// chart-upgrade) is a suite test method selected via the -run regex and runs as its +// own go test invocation, so all cross-phase state lives in-cluster under the +// deterministic names defined here. +package rancherupgrade + +const ( + // UpgradeInputConfigKey is the cattle-config key holding upgrade-phase inputs. + UpgradeInputConfigKey = "rancherUpgradeInput" + + // PipelineConfigKey marks a config as authored by the rancher-upgrade + // pipeline; generic PIT legs never set it. + PipelineConfigKey = "rancherUpgradePipeline" +) + +// UpgradeInput holds per-run upgrade parameters. TargetVersion is the Rancher +// version the server is upgraded to; written into the config by the pipeline +// (qa-infra-automation resolver output) or by hand for local runs. +type UpgradeInput struct { + TargetVersion string `json:"targetVersion" yaml:"targetVersion"` +} + +// PipelineInput is the marker section the rancher-upgrade pipeline writes into +// its cattle-config; the suite fails rather than skips when it is present but +// the upgrade target is missing (pipeline bug, not a generic PIT leg). +type PipelineInput struct { + Enabled bool `json:"enabled" yaml:"enabled"` +} diff --git a/validation/rancherupgrade/schemas/pit_schemas.yaml b/validation/rancherupgrade/schemas/pit_schemas.yaml new file mode 100644 index 0000000000..a95dc71f59 --- /dev/null +++ b/validation/rancherupgrade/schemas/pit_schemas.yaml @@ -0,0 +1,61 @@ +- projects: [RANCHERINT] + suite: Rancher Upgrade/Longhorn + cases: + - title: "Longhorn pre-upgrade setup" + description: "Install Longhorn and seed a checksum workload on a Longhorn volume before the Rancher server upgrade" + automation: 2 + steps: + - action: "Create a project on the Rancher local cluster" + data: "" + expectedresult: "The project is created successfully" + position: 1 + - action: "Install the Longhorn chart at the latest version offered by the pre-upgrade Rancher" + data: "Chart: longhorn from the rancher-charts repo" + expectedresult: "Longhorn installs and its deployments and daemonsets become Ready" + position: 2 + - action: "Create the checksum workload on a Longhorn-backed PVC" + data: "1GiB PVC and nginx deployment named rancher-upgrade-longhorn with data file /auto-mnt/checksum-data" + expectedresult: "PVC is Bound, deployment rolls out, and the checksum data is written to the Longhorn volume" + position: 3 + custom_field: + "15": "TestLonghornPreUpgrade" + - title: "Longhorn post-upgrade validation" + description: "Verify Longhorn and its volume data survive the Rancher server upgrade" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Check the Longhorn chart installation state" + data: "Chart: longhorn in longhorn-system" + expectedresult: "The chart is still installed" + position: 2 + - action: "Verify the checksum workload on the Longhorn volume" + data: "PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" + expectedresult: "PVC is Bound and the sha256 of the data file matches the pre-upgrade content" + position: 3 + custom_field: + "15": "TestLonghornPostUpgrade" + - title: "Longhorn chart upgrade after Rancher upgrade" + description: "Upgrade Longhorn to the version offered by the upgraded Rancher catalog and verify workload data" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Resolve the latest Longhorn chart version from the cluster-scoped catalog" + data: "Chart: longhorn from the rancher-charts repo" + expectedresult: "The latest cluster-scoped chart version is resolved and differs from the installed pre-upgrade version" + position: 2 + - action: "Upgrade the Longhorn chart with retry" + data: "longhorn and longhorn-crd to the latest version" + expectedresult: "The longhorn chart reaches Deployed state and workloads become Ready" + position: 3 + - action: "Verify the chart version and the checksum workload" + data: "Chart version equals latest; PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" + expectedresult: "Installed chart version matches latest and the sha256 of the data file still matches the pre-upgrade content" + position: 4 + custom_field: + "15": "TestLonghornChartUpgrade"