From 36fcdf06459fce0338023f5f1eed5a0640ddf5f8 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 7 Oct 2026 21:41:03 -0500 Subject: [PATCH 01/21] Add rancherupgrade package: Longhorn pre/post/chart-upgrade validation Implements the Longhorn vertical of the Rancher server upgrade validation (rancher/tests#967, parent PRD #946). New validation/rancherupgrade package with a LonghornUpgradeTestSuite whose phase methods run as separate go test invocations selected via -run, one per pipeline stage: - TestLonghornPreUpgrade: create project, install the Longhorn chart at the cluster-scoped catalog latest, create a checksum workload (PVC + nginx deployment with deterministic names) on a Longhorn volume. - TestLonghornPostUpgrade: gate on the management server-version setting reaching rancherUpgradeInput.targetVersion, assert the Longhorn chart survived the server upgrade, verify the checksum workload data. - TestLonghornChartUpgrade: gate on server version, resolve latest from the cluster-scoped catalog, upgrade longhorn + longhorn-crd, assert the installed chart version, verify the checksum workload again. Supporting actions: - actions/clusters: WaitRancherVersion polls the server-version setting until it semver-equals the target, retrying transient read errors. - actions/storage: CreateChecksumWorkload/VerifyChecksumWorkload build and verify a deterministically named PVC/deployment pair whose sha256-verified data file must survive across processes; no session cleanup is registered by design. - actions/charts: UpgradeLonghornChart upgrades longhorn and longhorn-crd with the deletingConfirmationFlag value preserved and OpenAPI validation disabled, waits for Deployed, then gates on deployments and daemonsets. TearDownSuite deliberately skips session.Cleanup and the suite is admin-only: upgrade-subject resources must survive into later phases and the pipeline environment teardown is the cleanup. Both deviations are documented in the package README, along with the token-rotation and catalog-follower-restart notes for the pipeline. Includes schemas/pit_schemas.yaml for the three phase methods. Verified live against a 2.14.3 HA env with a node-driver downstream RKE2 cluster: pre-upgrade PASS (Longhorn 109.3.1+up1.11.2, checksum seeded), server upgraded to 2.15.2 via qa-infra-automation, post-upgrade PASS, chart upgrade PASS (109.3.1+up1.11.2 -> 110.1.0+up1.12.1, checksum intact). --- actions/charts/longhorn.go | 118 +++++++++ actions/clusters/local.go | 36 +++ actions/storage/checksum.go | 234 ++++++++++++++++++ validation/rancherupgrade/README.md | 82 ++++++ validation/rancherupgrade/longhorn_test.go | 145 +++++++++++ validation/rancherupgrade/rancherupgrade.go | 18 ++ .../rancherupgrade/schemas/pit_schemas.yaml | 61 +++++ 7 files changed, 694 insertions(+) create mode 100644 actions/storage/checksum.go create mode 100644 validation/rancherupgrade/README.md create mode 100644 validation/rancherupgrade/longhorn_test.go create mode 100644 validation/rancherupgrade/rancherupgrade.go create mode 100644 validation/rancherupgrade/schemas/pit_schemas.yaml diff --git a/actions/charts/longhorn.go b/actions/charts/longhorn.go index 89c8640cb3..c1191e3859 100644 --- a/actions/charts/longhorn.go +++ b/actions/charts/longhorn.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" "github.com/rancher/shepherd/clients/rancher" "github.com/rancher/shepherd/clients/rancher/catalog" shepherdCharts "github.com/rancher/shepherd/extensions/charts" @@ -100,6 +101,123 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma return err } +// UpgradeLonghornChart upgrades the longhorn and longhorn-crd charts to +// installOptions.Version on installOptions.Cluster and waits for Deployed state. +// No session cleanup is registered: the uninstall cleanup belongs to the install +// call of the same session, and upgrade-test sessions must not delete the chart. +func UpgradeLonghornChart(client *rancher.Client, installOptions *InstallOptions) error { + serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) + if err != nil { + return err + } + + registrySetting, err := client.Management.Setting.ByID(defaultRegistrySettingID) + if err != nil { + return err + } + + longhornChartUpgradeActionPayload := &PayloadOpts{ + InstallOptions: *installOptions, + Name: LonghornChartName, + Namespace: LonghornNamespace, + Host: serverSetting.Value, + DefaultRegistry: registrySetting.Value, + } + + chartUpgradeAction := newLonghornChartUpgradeAction(longhornChartUpgradeActionPayload) + + catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + bodyBytes, err := json.Marshal(chartUpgradeAction) + if err != nil { + return err + } + err = ChartActionWithRetry(context.TODO(), client, verbUpgrade, longhornChartUpgradeActionPayload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbUpgrade, bodyBytes)) + if err != nil { + return err + } + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + if err != nil { + return err + } + + adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + watchAppInterface, err := adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + LonghornChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusPendingUpgrade) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + watchAppInterface, err = adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + LonghornChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusDeployed) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{}) + if err != nil { + return err + } + + return shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{}) +} + +// newLonghornChartUpgradeAction is a private helper function that returns the chart upgrade action for Longhorn. +func newLonghornChartUpgradeAction(p *PayloadOpts) *types.ChartUpgradeAction { + // Pass the deletingConfirmationFlag value on the main chart so the upgrade does + // not reset it; the CRD chart takes no Longhorn settings (same split as install). + chartUpgrade := NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, enableDeletionSetting) + chartUpgradeCRD := NewChartUpgrade(p.Name+"-crd", p.Name+"-crd", p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, nil) + chartUpgrades := []types.ChartUpgrade{*chartUpgradeCRD, *chartUpgrade} + + chartUpgradeAction := NewChartUpgradeAction(p.Namespace, chartUpgrades) + // Disable OpenAPI validation to avoid schema-conflict failures on the large + // Longhorn CRDs, which frequently change shape between versions. + chartUpgradeAction.DisableOpenAPIValidation = true + + return chartUpgradeAction +} + // UninstallLonghornChart removes Longhorn from the cluster related to the received catalog client object. func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID string, rancherHostname string) error { catalogClient, err := client.GetClusterCatalogClient(clusterID) diff --git a/actions/clusters/local.go b/actions/clusters/local.go index 206e523b81..752df582ee 100644 --- a/actions/clusters/local.go +++ b/actions/clusters/local.go @@ -1,13 +1,18 @@ package clusters import ( + "context" "errors" "fmt" "regexp" "strings" + "time" "github.com/Masterminds/semver/v3" "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -37,6 +42,37 @@ func IsRancherVersionAbove(client *rancher.Client, minVersion string) (bool, err return serverVersion.GreaterThan(minimumVersion), nil } +// WaitRancherVersion polls the management server-version setting until it equals +// targetVersion (semver-compared) or timeout elapses. Gates post-upgrade assertions +// on Rancher server convergence; transient read errors during the upgrade rollout +// are retried, not returned. +func WaitRancherVersion(client *rancher.Client, targetVersion string, timeout time.Duration) error { + target, err := parseRancherVersion(targetVersion) + if err != nil { + return err + } + + return wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, timeout, true, func(ctx context.Context) (done bool, err error) { + serverVersionRawValue, err := client.Management.Setting.ByID(serverVersionSetting) + if err != nil { + // The server may be mid-restart during the upgrade rollout; retry the read. + return false, nil + } + + serverVersion, err := parseRancherVersion(serverVersionRawValue.Value) + if err != nil { + return false, nil + } + + if serverVersion.Equal(target) { + return true, nil + } + + logrus.Debugf("Rancher server version is %q, waiting for %q", serverVersionRawValue.Value, targetVersion) + return false, nil + }) +} + func parseRancherVersion(versionInput string) (*semver.Version, error) { trimmedVersion := strings.TrimSpace(strings.TrimPrefix(versionInput, "v")) if trimmedVersion == "" { diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go new file mode 100644 index 0000000000..82ab4d305c --- /dev/null +++ b/actions/storage/checksum.go @@ -0,0 +1,234 @@ +package storage + +import ( + "context" + "crypto/sha256" + "fmt" + "strings" + + "github.com/rancher/shepherd/clients/rancher" + steveV1 "github.com/rancher/shepherd/clients/rancher/v1" + "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/extensions/defaults/namespaces" + "github.com/rancher/shepherd/extensions/defaults/stevetypes" + "github.com/rancher/shepherd/extensions/kubeapi/cluster" + "github.com/rancher/shepherd/extensions/kubeconfig" + wloads "github.com/rancher/shepherd/extensions/workloads" + "github.com/rancher/tests/actions/kubeapi/volumes/persistentvolumeclaims" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" +) + +// ChecksumDataFile is the filename written under MountPath by CreateChecksumWorkload. +const ChecksumDataFile = "checksum-data" + +// CreateChecksumWorkload creates a 1GiB PVC named on storageClassName in the +// default namespace, mounts it into a single-replica nginx deployment named at +// MountPath, waits for rollout, then writes content to MountPath/ChecksumDataFile via +// kubectl exec. Deterministic names let a later test phase (running as a separate +// process) relocate the resources. content must be shell-safe (alphanumeric/hyphen); +// it is interpolated into an sh -c command. No session cleanup is registered: callers +// are upgrade tests whose resources must outlive the process. +func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageClassName string, name string, content string) error { + storageClass, err := GetStorageClass(client, clusterID, storageClassName) + if err != nil { + return err + } + + wrangler, err := cluster.GetClusterWranglerContext(client, clusterID) + if err != nil { + return err + } + + accessModes := []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + } + + // Deliberate deviation from CreatePVC: no PV-delete/Longhorn-volume cleanup funcs + // are registered, because these resources must survive into later upgrade phases. + _, err = persistentvolumeclaims.CreatePersistentVolumeClaim( + client, + clusterID, + name, + "test-pvc-volume", + namespaces.Default, + 1, + accessModes, + nil, + &storageClass, + ) + if err != nil { + return err + } + + pvc := &corev1.PersistentVolumeClaim{} + err = wait.PollUntilContextTimeout(context.Background(), pollInterval, defaults.OneMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + pvc, err = wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + return pvc.Status.Phase == persistentvolumeclaims.PersistentVolumeBoundStatus, nil + }) + if err != nil { + return err + } + logrus.Debugf("PVC %s is Bound to volume %s", pvc.Name, pvc.Spec.VolumeName) + + volMount := corev1.VolumeMount{ + MountPath: MountPath, + Name: name, + } + + podVol := corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: name, + }, + }, + } + + steveclient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return err + } + + containerTemplate := wloads.NewContainer(nginxName, nginxName, corev1.PullAlways, []corev1.VolumeMount{volMount}, []corev1.EnvFromSource{}, nil, nil, nil) + podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: name}, nil) + deployment := wloads.NewDeploymentTemplate(name, namespaces.Default, podTemplate, true, nil) + + _, err = steveclient.SteveType(stevetypes.Deployment).Create(deployment) + if err != nil { + return err + } + + err = charts.WatchAndWaitDeployments(client, clusterID, namespaces.Default, metav1.ListOptions{ + FieldSelector: "metadata.name=" + name, + }) + if err != nil { + return err + } + + pod, err := getChecksumPod(client, clusterID, name) + if err != nil { + return err + } + + restConfig, err := getDownstreamRestConfig(client, clusterID) + if err != nil { + return err + } + + writeCommand := []string{"sh", "-c", "printf '%s' '" + content + "' > " + MountPath + "/" + ChecksumDataFile} + logrus.Debugf("Writing %s/%s on pod %s", MountPath, ChecksumDataFile, pod.Name) + _, err = kubeconfig.KubectlExec(restConfig, pod.Name, namespaces.Default, writeCommand) + + return err +} + +// VerifyChecksumWorkload relocates the PVC and pod created by CreateChecksumWorkload, +// asserts the PVC is still Bound, and asserts the sha256 of MountPath/ChecksumDataFile +// inside the running pod equals sha256(content). +func VerifyChecksumWorkload(client *rancher.Client, clusterID string, name string, content string) error { + wrangler, err := cluster.GetClusterWranglerContext(client, clusterID) + if err != nil { + return err + } + + pvc, err := wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) + if err != nil { + return err + } + + if pvc.Status.Phase != persistentvolumeclaims.PersistentVolumeBoundStatus { + return fmt.Errorf("PVC %s is %s, expected %s", name, pvc.Status.Phase, persistentvolumeclaims.PersistentVolumeBoundStatus) + } + + pod, err := getChecksumPod(client, clusterID, name) + if err != nil { + return err + } + + restConfig, err := getDownstreamRestConfig(client, clusterID) + if err != nil { + return err + } + + checksumCommand := []string{"sha256sum", MountPath + "/" + ChecksumDataFile} + output, err := kubeconfig.KubectlExec(restConfig, pod.Name, namespaces.Default, checksumCommand) + if err != nil { + return err + } + + // The exec runs with a TTY, so the output carries \r line endings; strip them + // before extracting the hash (first whitespace-separated field). + outputFields := strings.Fields(strings.ReplaceAll(output.String(), "\r", "")) + if len(outputFields) == 0 { + return fmt.Errorf("empty sha256sum output on pod %s", pod.Name) + } + + actualChecksum := outputFields[0] + expectedChecksum := fmt.Sprintf("%x", sha256.Sum256([]byte(content))) + if actualChecksum != expectedChecksum { + return fmt.Errorf("checksum mismatch on pod %s: expected %s, got %s", pod.Name, expectedChecksum, actualChecksum) + } + + return nil +} + +// getChecksumPod returns the pod in the default namespace labeled with +// DeploymentIdentifierLabel == name, waiting until it exists and is Running. +func getChecksumPod(client *rancher.Client, clusterID string, name string) (*steveV1.SteveAPIObject, error) { + var checksumPod *steveV1.SteveAPIObject + err := wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, defaults.TwoMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + steveclient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return false, err + } + + podsResp, err := steveclient.SteveType(stevetypes.Pod).NamespacedSteveClient(namespaces.Default).List(nil) + if err != nil { + return false, err + } + + for i := range podsResp.Data { + pod := &podsResp.Data[i] + if pod.Labels[DeploymentIdentifierLabel] != name { + continue + } + + podStatus := &corev1.PodStatus{} + err = steveV1.ConvertToK8sType(pod.Status, podStatus) + if err != nil { + return false, err + } + + if podStatus.Phase != corev1.PodRunning { + logrus.Debugf("Pod %s is %s, waiting for Running", pod.Name, podStatus.Phase) + return false, nil + } + + checksumPod = pod + return true, nil + } + + return false, nil + }) + + return checksumPod, err +} + +// getDownstreamRestConfig builds a rest config for kubectl exec against the cluster. +func getDownstreamRestConfig(client *rancher.Client, clusterID string) (*rest.Config, error) { + kubeConfig, err := kubeconfig.GetKubeconfig(client, clusterID) + if err != nil { + return nil, err + } + + return (*kubeConfig).ClientConfig() +} diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md new file mode 100644 index 0000000000..7436559c6b --- /dev/null +++ b/validation/rancherupgrade/README.md @@ -0,0 +1,82 @@ +# Rancher Upgrade + +The `rancherupgrade` package validates upgrading the Rancher server (2.14.x -> latest) +with interoperability products installed. The first vertical is Longhorn: a pre-upgrade +phase installs the Longhorn chart and seeds a checksum workload on a Longhorn volume, +the Rancher server is then upgraded out-of-band (qa-infra-automation `make rancher-upgrade`), +and post-upgrade phases verify the server version, the surviving Longhorn installation, +the workload data, and a Longhorn chart upgrade to the version offered by the upgraded +Rancher's catalog. + +Each phase is a suite test method selected via the `-run` regex and runs as its own +`go test` invocation in its own pipeline stage. All cross-phase state lives in-cluster +under deterministic names (PVC/deployment `rancher-upgrade-longhorn`, data file +`/auto-mnt/checksum-data`), never in process memory. + +## Deliberate deviations + +Do not "fix" the following; both are required by the upgrade workflow (rancher/tests#967): + +- `TearDownSuite` does not call `session.Cleanup()` and the helpers register no cleanup + funcs: pre-phase resources (Longhorn chart, PVC, workload) must survive into the + later phases, which run as separate processes. The pipeline environment teardown is + the cleanup. +- The suite is admin-only: upgrade phases assert server-level state and the pipeline + runs them with the admin token. + +## Config + +```yaml +rancher: + host: + adminToken: + insecure: true + clusterName: "" # pre-provisioned by qa-infra-automation + +rancherUpgradeInput: + targetVersion: "" +``` + +`rancher.clusterName` selects the shared downstream cluster. `rancherUpgradeInput.targetVersion` +is required by the post-upgrade and chart-upgrade phases; the pipeline writes it from +the qa-infra-automation resolver output, or set it by hand for local runs. + +## Running + +Pre-upgrade (installs Longhorn at the 2.14-era latest and seeds the checksum workload; +run against the not-yet-upgraded Rancher): + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade" +``` + +Post-upgrade (after `make rancher-upgrade RANCHER_VERSION_TO_UPGRADE= RANCHER_CHART_REPO_FLAVOR=community|prime` +in qa-infra-automation, and with `rancherUpgradeInput.targetVersion` set): + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade" +``` + +Chart upgrade: + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade" +``` + +## Notes + +- The shared downstream RKE2 cluster is provisioned by qa-infra-automation and must be + sized for Longhorn plus the later NeuVector/Observability suites; this is an infra + concern and is not configurable here. +- The Rancher server upgrade itself is `make rancher-upgrade` in + [qa-infra-automation](https://github.com/rancher/qa-infra-automation); this package + only runs the pre- and post-upgrade assertions around it. +- `make rancher-upgrade` rotates the admin token and writes the new one to + `ansible/rancher/-ha/generated.tfvars`; the pipeline must copy it into the test + config's `rancher.adminToken` before the post-upgrade phases (the pre-upgrade token + is revoked by the upgrade). +- After a server upgrade the downstream cluster-scoped catalog follower can keep + serving the previous Rancher line's index (e.g. release-v2.14 after upgrading to + 2.15) until the downstream `cattle-cluster-agent` re-syncs. If the chart-upgrade + phase fails with "failed to find chart longhorn from the chart repo", restart the + downstream cluster-agent and retry; observed re-sync time ~4 minutes. diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go new file mode 100644 index 0000000000..379fb38b9e --- /dev/null +++ b/validation/rancherupgrade/longhorn_test.go @@ -0,0 +1,145 @@ +//go:build validation && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 + +package rancherupgrade + +import ( + "testing" + + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + "github.com/rancher/tests/actions/charts" + actionsClusters "github.com/rancher/tests/actions/clusters" + "github.com/rancher/tests/actions/projects" + "github.com/rancher/tests/actions/storage" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type LonghornUpgradeTestSuite struct { + suite.Suite + session *session.Session + client *rancher.Client + cluster *clusters.ClusterMeta +} + +const ( + // checksumWorkloadName is the deterministic name shared by the PVC and the nginx + // deployment created in the pre-upgrade phase and relocated by later phases. + checksumWorkloadName = "rancher-upgrade-longhorn" + + // checksumContent is the shell-safe data written to the checksum file on the + // Longhorn-backed volume before the Rancher upgrade and verified after it. + checksumContent = "rancher-upgrade-longhorn-checksum-v1" +) + +// loadUpgradeInput returns the rancherUpgradeInput section of the test config. +func loadUpgradeInput() *UpgradeInput { + upgradeInput := new(UpgradeInput) + config.LoadConfig(UpgradeInputConfigKey, upgradeInput) + + return upgradeInput +} + +func (s *LonghornUpgradeTestSuite) SetupSuite() { + testSession := session.NewSession() + s.session = testSession + + client, err := rancher.NewClient("", testSession) + require.NoError(s.T(), err) + s.client = client + + s.cluster, err = clusters.NewClusterMeta(client, client.RancherConfig.ClusterName) + require.NoError(s.T(), err) +} + +// TearDownSuite deliberately does not call session.Cleanup(): resources created in +// the pre-upgrade phase (Longhorn chart, PVC, workload) must survive into the +// post-upgrade and chart-upgrade phases, which run as separate go test processes. +// The pipeline environment teardown is the cleanup for these resources. +func (s *LonghornUpgradeTestSuite) TearDownSuite() { +} + +func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { + project, _, err := projects.CreateProjectAndNamespace(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + // Resolve the version from the cluster-scoped catalog and install that: the + // management view can offer versions the cluster-scoped shard does not serve + // yet (rancher-2.15-catalog-sharding-issue.md), and the chart action validates + // against the cluster-scoped index. The pre-phase still yields the 2.14-era + // version by construction, since it runs against the pre-upgrade Rancher. + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + version, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + err = charts.InstallLonghornChart(s.client, charts.PayloadOpts{ + Namespace: charts.LonghornNamespace, + Host: s.client.RancherConfig.Host, + InstallOptions: charts.InstallOptions{ + Cluster: s.cluster, + Version: version, + ProjectID: project.ID, + }, + }, nil) + require.NoError(s.T(), err) + + err = storage.CreateChecksumWorkload(s.client, s.cluster.ID, charts.LonghornStorageClass, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) +} + +func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { + cfg := loadUpgradeInput() + require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.True(s.T(), chart.IsAlreadyInstalled, "longhorn chart must still be installed after the Rancher server upgrade") + + err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) +} + +func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { + cfg := loadUpgradeInput() + require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + + // Resolve the latest version from the cluster-scoped catalog: the management + // view can report versions absent from the cluster-scoped index (see + // rancher-2.15-catalog-sharding-issue.md at the repo root). + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + latest, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + installOptions := &charts.InstallOptions{ + Cluster: s.cluster, + Version: latest, + } + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return charts.UpgradeLonghornChart(s.client, installOptions) + }) + require.NoError(s.T(), err) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.Equal(s.T(), latest, chart.ChartDetails.Spec.Chart.Metadata.Version) + + err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) +} + +func TestLonghornUpgradeTestSuite(t *testing.T) { + suite.Run(t, new(LonghornUpgradeTestSuite)) +} diff --git a/validation/rancherupgrade/rancherupgrade.go b/validation/rancherupgrade/rancherupgrade.go new file mode 100644 index 0000000000..a08cf104eb --- /dev/null +++ b/validation/rancherupgrade/rancherupgrade.go @@ -0,0 +1,18 @@ +// Package rancherupgrade validates the Rancher server upgrade path (2.14.x -> latest) +// with interoperability products installed. Each phase (pre-upgrade, post-upgrade, +// chart-upgrade) is a suite test method selected via the -run regex and runs as its +// own go test invocation, so all cross-phase state lives in-cluster under the +// deterministic names defined here. +package rancherupgrade + +const ( + // UpgradeInputConfigKey is the cattle-config key holding upgrade-phase inputs. + UpgradeInputConfigKey = "rancherUpgradeInput" +) + +// UpgradeInput holds per-run upgrade parameters. TargetVersion is the Rancher +// version the server is upgraded to; written into the config by the pipeline +// (qa-infra-automation resolver output) or by hand for local runs. +type UpgradeInput struct { + TargetVersion string `json:"targetVersion" yaml:"targetVersion"` +} diff --git a/validation/rancherupgrade/schemas/pit_schemas.yaml b/validation/rancherupgrade/schemas/pit_schemas.yaml new file mode 100644 index 0000000000..da764c080d --- /dev/null +++ b/validation/rancherupgrade/schemas/pit_schemas.yaml @@ -0,0 +1,61 @@ +- projects: [RANCHERINT] + suite: Rancher Upgrade/Longhorn + cases: + - title: "Longhorn pre-upgrade setup" + description: "Install Longhorn and seed a checksum workload on a Longhorn volume before the Rancher server upgrade" + automation: 2 + steps: + - action: "Create a project on the downstream cluster" + data: "" + expectedresult: "The project is created successfully" + position: 1 + - action: "Install the Longhorn chart at the latest version offered by the pre-upgrade Rancher" + data: "Chart: longhorn from the rancher-charts repo" + expectedresult: "Longhorn installs and its deployments and daemonsets become Ready" + position: 2 + - action: "Create the checksum workload on a Longhorn-backed PVC" + data: "1GiB PVC and nginx deployment named rancher-upgrade-longhorn with data file /auto-mnt/checksum-data" + expectedresult: "PVC is Bound, deployment rolls out, and the checksum data is written to the Longhorn volume" + position: 3 + custom_field: + "15": "TestLonghornPreUpgrade" + - title: "Longhorn post-upgrade validation" + description: "Verify Longhorn and its volume data survive the Rancher server upgrade" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Check the Longhorn chart installation state" + data: "Chart: longhorn in longhorn-system" + expectedresult: "The chart is still installed" + position: 2 + - action: "Verify the checksum workload on the Longhorn volume" + data: "PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" + expectedresult: "PVC is Bound and the sha256 of the data file matches the pre-upgrade content" + position: 3 + custom_field: + "15": "TestLonghornPostUpgrade" + - title: "Longhorn chart upgrade after Rancher upgrade" + description: "Upgrade Longhorn to the version offered by the upgraded Rancher catalog and verify workload data" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Resolve the latest Longhorn chart version from the cluster-scoped catalog" + data: "Chart: longhorn from the rancher-charts repo" + expectedresult: "The latest cluster-scoped chart version is resolved" + position: 2 + - action: "Upgrade the Longhorn chart with retry" + data: "longhorn and longhorn-crd to the latest version" + expectedresult: "Both charts reach Deployed state and workloads become Ready" + position: 3 + - action: "Verify the chart version and the checksum workload" + data: "Chart version equals latest; PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" + expectedresult: "Installed chart version matches latest and the sha256 of the data file still matches the pre-upgrade content" + position: 4 + custom_field: + "15": "TestLonghornChartUpgrade" From f56386578fd546f95c31da2cfd5e4c9cf5dc624e Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 09:23:49 -0500 Subject: [PATCH 02/21] Add validation/Jenkinsfile.rancher-upgrade.e2e six-stage upgrade orchestrator Dedicated pipeline for the Rancher server upgrade validation (issue #968, PRD #946), community leg: deploy 2.14-latest env, pre-upgrade Longhorn phase, in-place server upgrade via make rancher-upgrade with rotated-token re-injection, post-upgrade and chart-upgrade phases against one shared Qase run, destroy-always with failure-only diagnostics. Versions resolve at runtime via make rancher-resolve-version when the params are empty. Structurally cloned from validation/Jenkinsfile.e2e and the airgap Jenkinsfile call shapes; runs go-pit-validation-job per phase with -runs TestLonghornUpgradeTestSuite/ selectors. --- validation/Jenkinsfile.rancher-upgrade.e2e | 981 +++++++++++++++++++++ 1 file changed, 981 insertions(+) create mode 100755 validation/Jenkinsfile.rancher-upgrade.e2e diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e new file mode 100755 index 0000000000..b867c1c195 --- /dev/null +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -0,0 +1,981 @@ +#!groovy +/** + * Dedicated Rancher server upgrade validation pipeline (rancher/tests#968, PRD #946). + * + * Six-stage flow, community leg: + * 1. Deploys a 2.14-latest environment via qa-infra-automation + * (make backend-s3, workspace-new, infra-up, cluster, rancher) + * 2. Pre-upgrade phase: installs Longhorn + seeds a checksum workload + * (validation/rancherupgrade, TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade) + * 3. Upgrades the Rancher server in place via `make rancher-upgrade` + * and re-injects the rotated admin token into the test config + * 4. Post-upgrade phase: asserts server version, Longhorn survival, workload data + * 5. Chart-upgrade phase: upgrades Longhorn to the version offered by the + * upgraded Rancher's catalog and re-verifies the workload data + * 6. Always destroys the environment (infra-down AUTO_APPROVE=yes) + * + * Versions resolve at runtime when RANCHER_VERSION_FROM / RANCHER_VERSION_TO_UPGRADE + * are empty (qa-infra-automation `make rancher-resolve-version`); pins override. + * + * Structurally cloned from validation/Jenkinsfile.e2e (deploy flow, admin-token + * injection, Qase testrun lifecycle, destroy post) and validation/pipeline/ + * Jenkinsfile.airgap-rke2-tests (make.runTarget call shapes, teardown guard). + * Unlike Jenkinsfile.e2e, this pipeline has no parallel downstream fan-out: the + * three test phases run sequentially against one shared Qase test run because + * each phase depends on the previous one's in-cluster state. + * + * Consumes shared functions from qa-jenkins-library: + * make.runTarget (backend-s3, workspace-new, infra-up, cluster, rancher, + * rancher-resolve-version, rancher-upgrade, infra-down) + * airgap.standardCheckout + * infrastructure.parseAndSubstituteVars, infrastructure.writeConfig, + * infrastructure.generateWorkspaceName, infrastructure.archiveWorkspaceName, + * infrastructure.writeSshKey, infrastructure.detectPublicIp + * property.useWithProperties + * + * Admin token generation uses a direct ansible-playbook call inside + * rancher-go-test:latest (no make token target exists upstream yet). + */ + +def libraryBranch = env.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +library "qa-jenkins-library@${libraryBranch}" + +def DEFAULT_RANCHER_PASSWORD = 'X%S3bPyg@n8kWcRs675l#fH0' + +// Suite/method selectors passed as GOTEST_TESTCASE to the downstream executor. +// Each phase is its own go test invocation (-run regex), so cross-phase state +// lives in-cluster under deterministic names, never in process memory. +def PRE_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade"' +def POST_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade"' +def CHART_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade"' + +/** + * Shared teardown helper: re-renders tfvars from params, tears down tofu workspace + * via make infra-down AUTO_APPROVE=yes. + * Sets env.INFRA_CLEANED to 'true' on success. + */ +def destroyInfrastructure(String reason) { + def wsName = '' + try { + if (fileExists('workspace_name.txt')) { + wsName = readFile('workspace_name.txt')?.trim() + } + } catch (e) { + echo "Could not read workspace_name.txt: ${e.message}" + } + + if (!wsName || !env.INFRA_DIR || env.INFRA_CLEANED == 'true') { + echo "Skipping teardown (${reason}): no workspace, no infra dir, or already cleaned" + return + } + + try { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + // Re-render terraform.tfvars so tofu destroy can access AWS state + def tofuDir = "${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes" + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME, + 'RUNNER_PUBLIC_IP': env.RUNNER_PUBLIC_IP + ] + ) + infrastructure.writeConfig( + path: "${tofuDir}/terraform.tfvars", + content: terraformConfig + ) + make.runTarget(target: 'infra-down', dir: env.INFRA_DIR, makeArgs: "WORKSPACE=${wsName} AUTO_APPROVE=yes") + env.INFRA_CLEANED = 'true' + } + } catch (cleanupErr) { + echo "Teardown failed (${reason}): ${cleanupErr.message}" + } +} + +/** + * Failure-only diagnostics: capture cluster state through the rancher-infra-tools + * container (kubectl + helm + kubeconfig from the deploy) into workspace files + * and archive them. Best-effort; never masks the failure it reports on. + */ +def collectUpgradeDiagnostics() { + if (!env.INFRA_DIR) { + echo 'Skipping diagnostics: no infra checkout available' + return + } + try { + def workspace = pwd() + sh """ + docker run --rm --platform linux/amd64 \ + -v ${workspace}:/workspace \ + -e KUBECONFIG=/workspace/${env.INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml \ + -w /workspace \ + rancher-infra-tools:latest \ + sh -c ' + set +e + kubectl get nodes -o wide > diagnostics-nodes.txt 2>&1 + kubectl get pods -A > diagnostics-pods.txt 2>&1 + helm history rancher -n cattle-system > diagnostics-helm-history.txt 2>&1 + kubectl logs -n cattle-system -l app=rancher --tail=300 > diagnostics-rancher-logs.txt 2>&1 + kubectl logs -n cattle-cluster-system -l app=cattle-cluster-agent --tail=300 > diagnostics-agent-logs.txt 2>&1 + exit 0 + ' + """ + if (params.REPORT_ARTIFACTS) { + archiveArtifacts artifacts: 'diagnostics-*.txt', allowEmptyArchive: true + } + } catch (diagErr) { + echo "Diagnostics collection failed (non-fatal): ${diagErr.message}" + } +} + +pipeline { + agent any + + options { + ansiColor('xterm') + // Deploy + three sequential test phases + server upgrade + destroy + timeout(time: 240, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '30')) + } + + parameters { + // ── Repository parameters ────────────────────────────── + string( + name: 'QA_JENKINS_LIBRARY_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-jenkins-library to use' + ) + string( + name: 'TESTS_REPO_URL', + defaultValue: 'https://github.com/rancher/tests', + description: 'URL of rancher/tests repository' + ) + string( + name: 'TESTS_BRANCH', + defaultValue: 'main', + description: 'Branch of rancher/tests repository' + ) + string( + name: 'QA_INFRA_REPO_URL', + defaultValue: 'https://github.com/rancher/qa-infra-automation', + description: 'URL of qa-infra-automation repository' + ) + string( + name: 'QA_INFRA_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-infra-automation repository' + ) + + // ── Upgrade parameters ───────────────────────────────── + string( + name: 'RANCHER_VERSION_FROM', + defaultValue: '', + description: 'Source Rancher version to deploy (empty = resolve latest 2.14.x patch at runtime)' + ) + string( + name: 'RANCHER_VERSION_TO_UPGRADE', + defaultValue: '', + description: 'Target Rancher version for the server upgrade (empty = resolve latest released final at runtime)' + ) + string( + name: 'RANCHER_CHART_REPO_FLAVOR', + defaultValue: 'community', + description: 'Chart repo flavor for the upgrade (community|prime; prime is a later slice)' + ) + + // ── Infrastructure parameters ────────────────────────── + string( + name: 'HOSTNAME_PREFIX', + defaultValue: 'pitupgrade', + description: 'Hostname prefix for *.qa.rancher.space and other AWS resources' + ) + string( + name: 'RKE2_VERSION', + defaultValue: 'v1.35.1+rke2r1', + description: 'RKE2 version to deploy' + ) + password( + name: 'RANCHER_BOOTSTRAP_PASSWORD', + defaultValue: DEFAULT_RANCHER_PASSWORD, + description: 'Rancher bootstrap password for initial setup' + ) + password( + name: 'RANCHER_ADMIN_PASSWORD', + defaultValue: DEFAULT_RANCHER_PASSWORD, + description: 'Rancher admin password after bootstrap' + ) + booleanParam( + name: 'DESTROY_ON_FAILURE', + defaultValue: true, + description: 'Collect diagnostics and tear down infrastructure if pipeline fails' + ) + booleanParam( + name: 'DESTROY_AFTER_TESTS', + defaultValue: true, + description: 'Tear down infrastructure after tests complete' + ) + + // ── Terraform parameters ─────────────────────────────── + string( + name: 'S3_BUCKET_NAME', + defaultValue: 'jenkins-terraform-state-storage', + description: 'S3 bucket name where Terraform state is stored' + ) + string( + name: 'S3_BUCKET_REGION', + defaultValue: 'us-east-2', + description: 'AWS region where the S3 bucket is located' + ) + string( + name: 'S3_KEY_PREFIX', + defaultValue: 'terraform.tfstate', + description: 'S3 key prefix for the Terraform state files' + ) + text( + name: 'TERRAFORM_CONFIG', + description: 'Terraform tfvars for the cluster_nodes module (non-airgap)', + defaultValue: '''aws_access_key = "${AWS_ACCESS_KEY_ID}" +aws_secret_key = "${AWS_SECRET_ACCESS_KEY}" +aws_ami = "ami-0600e22b69546c15c" +instance_type = "t3a.2xlarge" +aws_vpc = "vpc-bfccf4d7" +aws_subnet = "subnet-ee8cac86" +ephemeral_sg_ingress_cidrs = ["${RUNNER_PUBLIC_IP}/32"] +airgap_setup = false +proxy_setup = false +aws_volume_size = 500 +aws_volume_type = "gp3" +aws_hostname_prefix = "${HOSTNAME_PREFIX}" +aws_region = "us-east-2" +aws_route53_zone = "qa.rancher.space" +aws_ssh_user = "ubuntu" +public_ssh_key = "ssh_public_key.pub" +private_ssh_key = "~/.ssh/${AWS_SSH_PEM_KEY_NAME}" +nodes = [ + { + count = 3 + role = ["etcd", "cp", "worker"] + }, + { + count = 1 + role = ["worker"] + } +]''' + ) + text( + name: 'ANSIBLE_VARIABLES', + description: 'Ansible group_vars for RKE2 install + Rancher Helm deploy (non-airgap). Rancher version keys resolve from the pre-upgrade (FROM) version; rancher_version_upgrade keys hold the resolved target the server is upgraded to via make rancher-upgrade.', + defaultValue: '''--- +kubernetes_version: ${RKE2_VERSION} +cni: calico +kubeconfig_file: "/workspace/qa-infra-automation/ansible/rke2/default/kubeconfig.yaml" +cert_manager_version: "1.20.0" +bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +password: "${RANCHER_ADMIN_PASSWORD}" +rancher_version: ${RANCHER_VERSION} +rancher_image_tag: ${RANCHER_IMAGE_TAG} +rancher_chart_repo_url: "${RANCHER_CHART_REPO_URL}" +upgrade_mode: false +rancher_version_upgrade: "${RANCHER_VERSION_UPGRADE}" +rancher_image_tag_upgrade: "${RANCHER_IMAGE_TAG_UPGRADE}" +rancher_chart_repo_upgrade: "${RANCHER_UPGRADE_REPO_NAME}" +k8s_upgrade_mode: false +server_flags: | + ingress-controller: traefik''' + ) + + // ── Test parameters ──────────────────────────────────── + text( + name: 'CATTLE_TEST_CONFIG', + description: 'Test configuration YAML (cattle-config). rancher.adminToken is injected at runtime (initial + rotated after the server upgrade); rancherUpgradeInput.targetVersion is substituted from the resolved upgrade target.', + defaultValue: '''rancher: + host: ${HOSTNAME_PREFIX}.qa.rancher.space + adminPassword: ${ADMIN_PASSWORD} + insecure: true + # Deliberately false: pre-upgrade resources (Longhorn chart, PVC, workload) + # must survive into the post-upgrade phases; the pipeline environment + # teardown is the cleanup. + cleanup: false + # The qa-infra-automation cluster_nodes RKE2 cluster is Rancher's own + # ("local") cluster; the upgrade suite installs Longhorn on it. + clusterName: local +rancherUpgradeInput: + targetVersion: "${RANCHER_VERSION_UPGRADE}"''', + ) + booleanParam( + name: 'REPORT_ARTIFACTS', + defaultValue: true, + description: 'Archive token metadata, Qase run config, and failure diagnostics as build artifacts' + ) + + // ── Downstream job parameters ───────────────────────── + string( + name: 'INDIVIDUAL_JOB', + defaultValue: 'go-pit-validation-job', + description: 'Name of the downstream Jenkins job that runs individual Go test suites' + ) + string( + name: 'TEST_PACKAGE', + defaultValue: './validation/rancherupgrade/...', + description: 'Go test package path passed to the downstream job' + ) + string( + name: 'TEST_TIMEOUT', + defaultValue: '60m', + description: 'Timeout for downstream test jobs' + ) + string( + name: 'QASE_REPORTER_SCRIPT', + defaultValue: 'build_qase_reporter_v2.sh', + description: 'Qase reporter script name in pipeline/scripts/' + ) + string( + name: 'QASE_SCHEMA_PREFIX', + defaultValue: 'pit', + description: 'Schema file prefix for Qase operations, for example hostbusters or pit' + ) + } + + environment { + WORKSPACE_NAME = '' + INFRA_CLEANED = 'false' + YQ_VERSION = 'v4.47.2' + // Folder-injected TEST_RUN_NAME (if present) wins; the fallback keeps + // the shared Qase run name readable when it is not injected. + TEST_RUN_NAME = "${env.TEST_RUN_NAME ?: 'Rancher Server Upgrade Validation'}" + } + + stages { + stage('Checkout') { + steps { + script { + def dirs = airgap.standardCheckout( + testsRepo: [url: params.TESTS_REPO_URL, branch: params.TESTS_BRANCH], + infraRepo: [url: params.QA_INFRA_REPO_URL, branch: params.QA_INFRA_BRANCH] + ) + env.TESTS_DIR = dirs.testsDir + env.INFRA_DIR = dirs.infraDir + + def hostnamePrefix = params.HOSTNAME_PREFIX ?: 'unknown' + currentBuild.displayName = "#${env.BUILD_NUMBER} - ${hostnamePrefix}" + currentBuild.description = "Rancher server upgrade validation for ${hostnamePrefix}" + } + } + } + + stage('Build Docker Images') { + steps { + // rancher-infra-tools:latest — used by make.groovy / infra.* for all make targets + sh "docker build --no-cache --platform linux/amd64 -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." + // rancher-go-test:latest — used for qase binary build and admin-token playbook + sh "docker build --no-cache --platform linux/amd64 -t rancher-go-test:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.airgap-go-tests ." + } + } + + stage('Setup Tools') { + steps { + sh """ + wget -qO ./yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + wget -qO ./yq_checksums https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/checksums + grep -q '^yq_linux_amd64 ' ./yq_checksums + awk '/^yq_linux_amd64 / {print \$19 \" ./yq\"}' ./yq_checksums | sha256sum -c - + rm -f ./yq_checksums + chmod +x ./yq + """ + } + } + + // Auto-detect this runner's public IP so the ephemeral security + // group's SSH/LB ingress rule (ephemeral_sg_ingress_cidrs) allows + // the IP that Ansible/SSH will actually originate from. + stage('Detect Public IP') { + steps { + script { + env.RUNNER_PUBLIC_IP = infrastructure.detectPublicIp() + } + } + } + + // ── Resolve the upgrade window ───────────────────────── + // Empty params resolve at runtime via qa-infra-automation's + // rancher-resolve-version target; explicit pins pass through. + // The chosen lines/regexes and resolved versions are echoed so CI + // logs show exactly which version each stage ran against. + stage('Resolve Rancher Versions') { + steps { + script { + env.RANCHER_FROM_VERSION = params.RANCHER_VERSION_FROM?.trim() ?: '' + env.RANCHER_TO_VERSION = params.RANCHER_VERSION_TO_UPGRADE?.trim() ?: '' + + if (!env.RANCHER_FROM_VERSION) { + echo 'RANCHER_VERSION_FROM empty — resolving latest 2.14.x patch (SOURCE_LINE=2.14, latest final of the line)' + def resolveOut = make.runTarget( + target: 'rancher-resolve-version', + dir: env.INFRA_DIR, + makeArgs: ['SOURCE_LINE=2.14'], + returnStdout: true, + passAwsCreds: false + ) + writeFile file: 'resolve-from.json', text: resolveOut + env.RANCHER_FROM_VERSION = sh( + script: "./yq '.chart_version' resolve-from.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + env.RANCHER_FROM_IMAGE_TAG = sh( + script: "./yq '.image_tag' resolve-from.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + } else { + env.RANCHER_FROM_IMAGE_TAG = env.RANCHER_FROM_VERSION.startsWith('v') ? env.RANCHER_FROM_VERSION : "v${env.RANCHER_FROM_VERSION}" + echo "RANCHER_VERSION_FROM pinned: ${env.RANCHER_FROM_VERSION} (image tag ${env.RANCHER_FROM_IMAGE_TAG})" + } + + if (!env.RANCHER_TO_VERSION) { + echo 'RANCHER_VERSION_TO_UPGRADE empty — resolving latest released final (no SOURCE_LINE, repo index latest final)' + def resolveOut = make.runTarget( + target: 'rancher-resolve-version', + dir: env.INFRA_DIR, + returnStdout: true, + passAwsCreds: false + ) + writeFile file: 'resolve-to.json', text: resolveOut + env.RANCHER_TO_VERSION = sh( + script: "./yq '.chart_version' resolve-to.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + env.RANCHER_TO_IMAGE_TAG = sh( + script: "./yq '.image_tag' resolve-to.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + } else { + env.RANCHER_TO_IMAGE_TAG = env.RANCHER_TO_VERSION.startsWith('v') ? env.RANCHER_TO_VERSION : "v${env.RANCHER_TO_VERSION}" + echo "RANCHER_VERSION_TO_UPGRADE pinned: ${env.RANCHER_TO_VERSION} (image tag ${env.RANCHER_TO_IMAGE_TAG})" + } + + echo """ + Upgrade window resolved: + FROM (deploy): ${env.RANCHER_FROM_VERSION} (${env.RANCHER_FROM_IMAGE_TAG}) + TO (upgrade): ${env.RANCHER_TO_VERSION} (${env.RANCHER_TO_IMAGE_TAG}) + Chart flavor: ${params.RANCHER_CHART_REPO_FLAVOR} + """.stripIndent() + currentBuild.description = "Upgrade ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (${params.RANCHER_CHART_REPO_FLAVOR}) | ${params.HOSTNAME_PREFIX}" + } + } + } + + stage('Deploy Rancher Pre-Upgrade Environment') { + steps { + script { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/default" + def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes" + def workspaceName = '' + + // ── SSH key setup ────────────────────────── + stage('Configure SSH Key') { + infrastructure.writeSshKey( + keyContent: env.AWS_SSH_PEM_KEY, + keyName: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ) + // Copy public key into the tofu module directory so + // tofu's file() function can read it (restricted to + // the module directory tree). + sh "cp .ssh/*.pub ${tofuModulePath}/ssh_public_key.pub" + } + + // ── Render terraform.tfvars ───────────────── + // Must exist before any make target runs (check-tofu-dir validates it). + stage('Configure Tofu Variables') { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME, + 'RUNNER_PUBLIC_IP': env.RUNNER_PUBLIC_IP + ] + ) + infrastructure.writeConfig( + path: "${tofuModulePath}/terraform.tfvars", + content: terraformConfig + ) + } + + // ── Initialize S3 backend ─────────────────── + stage('Initialize Tofu Backend') { + make.runTarget( + target: 'backend-s3', + dir: env.INFRA_DIR, + makeArgs: [ + "BUCKET=${params.S3_BUCKET_NAME}", + "KEY=${params.S3_KEY_PREFIX}", + "REGION=${params.S3_BUCKET_REGION}" + ] + ) + } + + // ── Create tofu workspace ─────────────────── + stage('Create Workspace') { + workspaceName = infrastructure.generateWorkspaceName( + prefix: 'jenkins_rancher_upgrade_workspace', + suffix: params.HOSTNAME_PREFIX, + includeTimestamp: false + ) + make.runTarget( + target: 'workspace-new', + dir: env.INFRA_DIR, + makeArgs: "WORKSPACE=${workspaceName}" + ) + infrastructure.archiveWorkspaceName(workspaceName: workspaceName) + env.WORKSPACE_NAME = workspaceName + } + + // ── Provision infra + generate inventory ──── + stage('Provision Infrastructure') { + make.runTarget( + target: 'infra-up', + dir: env.INFRA_DIR, + makeArgs: "WORKSPACE=${workspaceName}" + ) + } + + // ── Write group_vars/all.yml ──────────────── + // Must exist before make cluster (make infra-up generates + // inventory.yml but not the group_vars subdirectory). + // Rancher version keys carry the resolved FROM version; + // the *_upgrade keys carry the resolved TO version the + // later `make rancher-upgrade` stage targets. + stage('Configure Ansible Variables') { + sh "mkdir -p ${ansiblePath}/inventory/group_vars" + + // The FROM version is a stable semver patch line, so + // its chart repo is the stable "latest" index. + def fromChartRepoUrl = 'https://releases.rancher.com/server-charts/latest' + def upgradeRepoName = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'rancher-prime-upgrade' : 'rancher-upgrade' + + def ansibleVars = infrastructure.parseAndSubstituteVars( + content: params.ANSIBLE_VARIABLES, + envVars: [ + 'RKE2_VERSION': params.RKE2_VERSION, + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_VERSION': env.RANCHER_FROM_VERSION, + 'RANCHER_IMAGE_TAG': env.RANCHER_FROM_IMAGE_TAG, + 'RANCHER_CHART_REPO_URL': fromChartRepoUrl, + 'RANCHER_VERSION_UPGRADE': env.RANCHER_TO_VERSION, + 'RANCHER_IMAGE_TAG_UPGRADE': env.RANCHER_TO_IMAGE_TAG, + 'RANCHER_UPGRADE_REPO_NAME': upgradeRepoName + ] + ) + + def rancherVars = infrastructure.parseAndSubstituteVars( + content: '''deploy_rancher: true +install_helm: true +rancher_hostname: "${HOSTNAME_PREFIX}.qa.rancher.space" +rancher_bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +rancher_admin_password: "${RANCHER_ADMIN_PASSWORD}" +rancher_use_bundled_system_charts: true +''', + envVars: [ + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD + ] + ) + + // Merge ANSIBLE_VARIABLES + Rancher vars into a single group_vars/all.yml + infrastructure.writeConfig( + path: "${ansiblePath}/inventory/group_vars/all.yml", + content: ansibleVars + '\n' + rancherVars + ) + } + + // ── Deploy RKE2 cluster (ansible; no AWS creds) ── + stage('Deploy RKE2 Cluster') { + make.runTarget( + target: 'cluster', + dir: env.INFRA_DIR, + passAwsCreds: false + ) + } + + // ── Deploy pre-upgrade Rancher (helm; no AWS creds) ── + stage('Deploy Rancher') { + make.runTarget( + target: 'rancher', + dir: env.INFRA_DIR, + passAwsCreds: false + ) + } + + // ── Capture Rancher hostname ──────────────── + stage('Output Infrastructure Details') { + // Rancher hostname is deterministically constructed from HOSTNAME_PREFIX + // matching the value set in rancherVars (rancher_hostname). + def rancherHostname = "${params.HOSTNAME_PREFIX}.qa.rancher.space" + echo "Rancher Hostname: https://${rancherHostname}" + env.RANCHER_HOSTNAME = rancherHostname + } + } + } + } + } + + stage('Create Test Config and Inject Admin Token') { + steps { + script { + def workspace = pwd() + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + + // Create cattle-config with variable substitution, including + // the resolved upgrade target for rancherUpgradeInput.targetVersion. + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY' + ]) { + def resolvedConfig = infrastructure.parseAndSubstituteVars( + content: params.CATTLE_TEST_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_VERSION_UPGRADE': env.RANCHER_TO_VERSION + ] + ) + writeFile file: cattleConfigPath, text: resolvedConfig.trim() + '\n' + } + + // Generate admin token via Ansible playbook + // TODO: replace with 'make token' once that target is added upstream + def containerConfigPath = "/workspace/${cattleConfigPath}" + def tokenTtl = env.RANCHER_TOKEN_TTL ?: '0' + def tokenDescription = "jenkins-rancher-upgrade-${env.BUILD_NUMBER}" + + // Diagnostic logging to validate env + output path before token generation + echo "Token generation context: workspace=${workspace}, config=${containerConfigPath}" + echo "RANCHER_ADMIN_PASSWORD present in params: ${params.RANCHER_ADMIN_PASSWORD ? 'yes' : 'no'}" + + sh """ + docker run --rm --platform linux/amd64 \ + --name generate-token \ + -e RANCHER_ADMIN_PASSWORD \ + -e RANCHER_URL=https://${params.HOSTNAME_PREFIX}.qa.rancher.space \ + -e ANSIBLE_CONFIG=/workspace/${env.INFRA_DIR}/ansible/ansible.cfg \ + -v ${workspace}:/workspace \ + -w /workspace/${env.INFRA_DIR}/ansible/rke2/default \ + rancher-go-test:latest \ + ansible-playbook -i inventory/inventory.yml /workspace/${env.INFRA_DIR}/ansible/rancher/token/generate-admin-token.yml \ + -e rancher_cattle_config_file=${containerConfigPath} \ + -e rancher_token_ttl=${tokenTtl} \ + -e rancher_token_description=${tokenDescription} \ + -e rancher_token_output_format=json \ + -e rancher_token_output_file=/workspace/rancher-token.json \ + -e rancher_url=https://${params.HOSTNAME_PREFIX}.qa.rancher.space + """ + + def tokenFileExists = fileExists('rancher-token.json') + echo "Token file present after generation: ${tokenFileExists}" + + if (tokenFileExists) { + // Redact secret fields from token JSON via Docker container. + // Pipeline Utility Steps plugin (readJSON/writeJSON) is not installed, + // and python3 is not available on the Jenkins agent host. + sh "docker run --rm -v \$(pwd):/workspace -w /workspace rancher-go-test:latest python3 -c \"import json; d=json.load(open('rancher-token.json')); [d.pop(k,None) for k in ('token','bearerToken','accessKey','secretKey','value')]; json.dump(d,open('rancher-token-metadata.json','w'),indent=2)\"" + if (params.REPORT_ARTIFACTS) { + archiveArtifacts artifacts: 'rancher-token-metadata.json' + echo 'Archived redacted token metadata: rancher-token-metadata.json' + } + + // Infrastructure, Rancher deploy, and admin token all succeeded. + // Shared library helpers (infra.*) may have set currentBuild.result to + // UNSTABLE/FAILURE via internal catchError() wrappers for non-fatal warnings, + // which would cause declarative pipeline to skip downstream test stages. + // Reset build result to SUCCESS so the upgrade phases can execute. + if (currentBuild.result && currentBuild.result != 'SUCCESS') { + echo "Resetting currentBuild.result from '${currentBuild.result}' to 'SUCCESS' — token generation succeeded, allowing upgrade phases to run" + currentBuild.result = 'SUCCESS' + } + } else { + error 'rancher-token.json not found after token generation — cannot run test phases without an admin token' + } + } + } + } + + // ── Phase 1: pre-upgrade Longhorn install + workload seed ── + // Starts the shared Qase run once; all three phases report into it. + stage('Pre-Upgrade Longhorn Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def workspace = pwd() + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + def testRunConfig = "${workspace}/testrunconfig.yaml" + def qaseBinary = "${workspace}/testrun" + + // Build qase testrun binary into the workspace root so the + // agent-side invocation below and testrunconfig.yaml (written + // to the binary's CWD) line up. + sh """ + docker run --rm --platform linux/amd64 \ + -v ${workspace}:/workspace \ + -w /workspace/${env.TESTS_DIR} \ + rancher-go-test:latest \ + sh -c 'env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /workspace/testrun ./validation/pipeline/qase/testrun' + """ + + // Start the shared qase test run (completed after the final phase) + sh "${qaseBinary} --startRun=true" + + // Read test run ID + def testRunID = '' + if (fileExists(testRunConfig)) { + testRunID = sh( + script: "./yq '.id' ${testRunConfig}", + returnStdout: true + ).trim() + } + echo "Shared Qase test run started: id=${testRunID}" + if (params.REPORT_ARTIFACTS && fileExists(testRunConfig)) { + archiveArtifacts artifacts: 'testrunconfig.yaml', allowEmptyArchive: true + } + + // NOTE: unlike Jenkinsfile.e2e, rancher.cleanup is NOT set to + // true and no desiredflags are forced here — the upgrade suite's + // cross-phase state must survive, and the pipeline environment + // teardown is the cleanup. + def configYaml = readFile(file: cattleConfigPath) + + echo "Dispatching pre-upgrade phase: GOTEST_TESTCASE ${PRE_UPGRADE_TESTCASE}" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: PRE_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } + } + } + } + + // ── Phase 2: in-place Rancher server upgrade ── + // `make rancher-upgrade` rotates the Rancher admin token and writes the + // new one to ansible/rancher/default-ha/generated.tfvars; the old token + // is revoked, so the test config MUST be re-injected before the + // post-upgrade phases dispatch (verified live: old token 401s). + stage('Upgrade Rancher Server') { + steps { + script { + echo "Upgrading Rancher server: ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (flavor ${params.RANCHER_CHART_REPO_FLAVOR})" + make.runTarget( + target: 'rancher-upgrade', + dir: env.INFRA_DIR, + makeArgs: [ + "ENV=default", + "RANCHER_VERSION_TO_UPGRADE=${env.RANCHER_TO_VERSION}", + "RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}" + ], + passAwsCreds: false + ) + + // Re-inject the rotated admin token into the test config + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + def generatedTfvars = "${env.INFRA_DIR}/ansible/rancher/default-ha/generated.tfvars" + + if (!fileExists(generatedTfvars)) { + error "Rotated token file not found at ${generatedTfvars} — post-upgrade phases cannot authenticate" + } + def rotatedToken = sh( + script: "awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}", + returnStdout: true + ).trim() + if (!rotatedToken) { + error "Could not extract api_key from ${generatedTfvars} — post-upgrade phases cannot authenticate" + } + echo "Rotated admin token extracted (length ${rotatedToken.length()}); re-injecting into test config" + + // The token never leaves this shell step (no Groovy/env + // interpolation), so it cannot leak into the build log. + sh """ + token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) + ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} + """ + echo "Test config now carries the post-upgrade admin token; rancherUpgradeInput.targetVersion=${env.RANCHER_TO_VERSION}" + } + } + } + + // ── Phase 3: post-upgrade assertions ── + stage('Post-Upgrade Longhorn Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def testRunID = sh( + script: "./yq '.id' testrunconfig.yaml", + returnStdout: true + ).trim() + def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + + echo "Dispatching post-upgrade phase: GOTEST_TESTCASE ${POST_UPGRADE_TESTCASE} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: POST_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } + } + } + } + + // ── Phase 4: Longhorn chart upgrade through the upgraded catalog ── + // Completes the shared Qase run after this final phase; do not rely on + // the downstream job's own QASE_TEST_RUN_COMPLETE for the shared run. + stage('Longhorn Chart Upgrade Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def testRunID = sh( + script: "./yq '.id' testrunconfig.yaml", + returnStdout: true + ).trim() + def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + + echo "Dispatching chart-upgrade phase: GOTEST_TESTCASE ${CHART_UPGRADE_TESTCASE} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: CHART_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + + // Complete the shared qase test run + sh "./testrun --startRun=false" + env.QASE_RUN_COMPLETED = 'true' + echo "Shared Qase test run ${testRunID} completed" + } + } + } + } + } + + post { + failure { + script { + if (params.DESTROY_ON_FAILURE) { + collectUpgradeDiagnostics() + destroyInfrastructure('DESTROY_ON_FAILURE') + } + } + } + + always { + script { + // Complete the shared Qase run when a phase failed before the + // final stage could close it (no-op once already completed). + if (env.QASE_RUN_COMPLETED != 'true' && fileExists('testrun') && fileExists('testrunconfig.yaml')) { + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + sh './testrun --startRun=false' + } + env.QASE_RUN_COMPLETED = 'true' + } + } + + if (params.DESTROY_AFTER_TESTS) { + destroyInfrastructure('DESTROY_AFTER_TESTS') + } + + // Docker cleanup + sh 'docker rmi -f rancher-go-test:latest rancher-infra-tools:latest || true' + + // Sensitive artifact cleanup — shred or remove credentials from workspace + sh ''' + set +e + + cleanup_file() { + if [ -f "$1" ]; then + if command -v shred >/dev/null 2>&1; then + shred -u "$1" || rm -f "$1" + else + rm -f "$1" + fi + fi + } + + cleanup_dir() { + if [ -d "$1" ]; then + find "$1" -type f -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + rm -rf "$1" + fi + } + + cleanup_file ".go_test_env" + cleanup_file "rancher-token.json" + cleanup_file "rancher-token-metadata.json" + cleanup_file "workspace_name.txt" + cleanup_file "resolve-from.json" + cleanup_file "resolve-to.json" + + if [ -n "${INFRA_DIR}" ] && [ -d "${INFRA_DIR}" ]; then + find "${INFRA_DIR}" -type f \\( -name 'terraform.tfvars' -o -name 'generated.tfvars' -o -name '*.pem' -o -name 'id_rsa' -o -name 'id_ed25519' \\) -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + fi + + cleanup_dir ".ssh" + ''' + } + } + } +} From 3e14ff9192b91826a0eaf132e66c73063a085cbc Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 10:38:56 -0500 Subject: [PATCH 03/21] Add awsNodeTemplate to upgrade cattle-config for SG authorization The per-phase executor job can land on a different Jenkins agent than the orchestrator (observed in the first rancher-upgrade run: default-n367q vs default-xj00w). The LB ingress on the tofu-created ephemeral SG only allows the orchestrator's detected public IP, so a phase on another agent could not reach the Rancher API and rancher.NewClient timed out after 22 minutes of retries. awsNodeTemplate carries the three keys the executor's Authorize Security Group stage reads (securityGroup, vpcId, region) so it authorizes its own runner IP on tf--sg before dispatching to the API, matching the shape Jenkinsfile.e2e ships for the same reason. --- validation/Jenkinsfile.rancher-upgrade.e2e | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index b867c1c195..8ac1006043 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -295,7 +295,17 @@ server_flags: | text( name: 'CATTLE_TEST_CONFIG', description: 'Test configuration YAML (cattle-config). rancher.adminToken is injected at runtime (initial + rotated after the server upgrade); rancherUpgradeInput.targetVersion is substituted from the resolved upgrade target.', - defaultValue: '''rancher: + defaultValue: '''# The downstream per-phase executor can land on a different Jenkins agent +# than this orchestrator; awsNodeTemplate carries the three keys its +# Authorize Security Group stage needs (sg name, vpc, region) so the +# runner's public IP gets authorized on the tofu-created ephemeral SG +# (tf--sg) before the phases talk to the Rancher LB. +awsNodeTemplate: + region: us-east-2 + vpcId: vpc-bfccf4d7 + securityGroup: + - tf-${HOSTNAME_PREFIX}-sg +rancher: host: ${HOSTNAME_PREFIX}.qa.rancher.space adminPassword: ${ADMIN_PASSWORD} insecure: true From 90381bdcbb5bc4f28253f16f1021d617fb05ff2c Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 11:41:09 -0500 Subject: [PATCH 04/21] Make repo/executor params sticky against declarative parameter sync A declarative pipeline re-applies its parameters{} block to the job definition after each run, which silently overwrote the JJB-configured staging values (fork repos + slice branches) with the hard-coded main defaults on the mower job: the first run deployed from the fork, the next one fell back to upstream main and lost make rancher-resolve-version ('No rule to make target'). Defaulting these five params to their current value when set keeps the per-controller JJB configuration authoritative instead of racing it. --- validation/Jenkinsfile.rancher-upgrade.e2e | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 8ac1006043..80a19180be 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -152,24 +152,29 @@ pipeline { defaultValue: 'main', description: 'Branch of qa-jenkins-library to use' ) + // Repo params keep whatever the job was last triggered with: a + // declarative pipeline re-applies its parameters{} block to the job + // after each run, and hard-coded defaults here would clobber the + // JJB-configured per-controller values (fork + slice branches on the + // staging controller, upstream main on the main controller). string( name: 'TESTS_REPO_URL', - defaultValue: 'https://github.com/rancher/tests', + defaultValue: params.TESTS_REPO_URL ?: 'https://github.com/rancher/tests', description: 'URL of rancher/tests repository' ) string( name: 'TESTS_BRANCH', - defaultValue: 'main', + defaultValue: params.TESTS_BRANCH ?: 'main', description: 'Branch of rancher/tests repository' ) string( name: 'QA_INFRA_REPO_URL', - defaultValue: 'https://github.com/rancher/qa-infra-automation', + defaultValue: params.QA_INFRA_REPO_URL ?: 'https://github.com/rancher/qa-infra-automation', description: 'URL of qa-infra-automation repository' ) string( name: 'QA_INFRA_BRANCH', - defaultValue: 'main', + defaultValue: params.QA_INFRA_BRANCH ?: 'main', description: 'Branch of qa-infra-automation repository' ) @@ -328,7 +333,7 @@ rancherUpgradeInput: // ── Downstream job parameters ───────────────────────── string( name: 'INDIVIDUAL_JOB', - defaultValue: 'go-pit-validation-job', + defaultValue: params.INDIVIDUAL_JOB ?: 'go-pit-validation-job', description: 'Name of the downstream Jenkins job that runs individual Go test suites' ) string( From 263293c65c92f8f408c0a34755e89dc778d9873e Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 12:26:04 -0500 Subject: [PATCH 05/21] Wait up to ten minutes for the checksum PVC to bind The one-minute bind wait raced a freshly installed Longhorn: the first volume's provisioning (engine-image pull, instance-manager startup, CSI registration) can exceed a minute on a cold cluster, which failed the pre-upgrade phase with 'context deadline exceeded' in the mower run (the same step passed when the volume bound faster). Ten minutes matches the bound used for the rest of the storage waits. --- actions/storage/checksum.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go index 82ab4d305c..9f3d173cf7 100644 --- a/actions/storage/checksum.go +++ b/actions/storage/checksum.go @@ -66,7 +66,10 @@ func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageCla } pvc := &corev1.PersistentVolumeClaim{} - err = wait.PollUntilContextTimeout(context.Background(), pollInterval, defaults.OneMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + // TenMinuteTimeout: the first volume on a freshly installed Longhorn waits + // on engine-image pull, instance-manager startup, and CSI registration, + // which routinely exceeds a minute on a cold cluster. + err = wait.PollUntilContextTimeout(context.Background(), pollInterval, defaults.TenMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { pvc, err = wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) if err != nil { return false, err From bd78700ce424faaa65add61729f9b5a79a793433 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 12:51:36 -0500 Subject: [PATCH 06/21] Collect failure diagnostics before destroy and image cleanup Jenkins evaluates declarative post conditions in a fixed order with always before failure, so the failure-block diagnostics ran after the always block had already run infra-down and docker rmi: kubectl/helm had no rancher-infra-tools image left to run in (pull access denied, exit 125) and the cluster was already gone regardless. Diagnostics now run first inside always, gated on a non-SUCCESS result, while the images and the live cluster still exist; the failure block keeps only the DESTROY_ON_FAILURE fallback destroy. --- validation/Jenkinsfile.rancher-upgrade.e2e | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 80a19180be..0993fde1ee 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -914,7 +914,6 @@ rancher_use_bundled_system_charts: true failure { script { if (params.DESTROY_ON_FAILURE) { - collectUpgradeDiagnostics() destroyInfrastructure('DESTROY_ON_FAILURE') } } @@ -922,6 +921,15 @@ rancher_use_bundled_system_charts: true always { script { + // Failure diagnostics run here, not in the failure block: + // Jenkins evaluates post conditions in a fixed order with + // always first, so anything in failure{} would already have + // lost the docker images (rmi below) and the live cluster + // (destroy below). Collect while both still exist. + if (currentBuild.currentResult != 'SUCCESS') { + collectUpgradeDiagnostics() + } + // Complete the shared Qase run when a phase failed before the // final stage could close it (no-op once already completed). if (env.QASE_RUN_COMPLETED != 'true' && fileExists('testrun') && fileExists('testrunconfig.yaml')) { From c1237a56ac5fb2c00618e91eb11b790e657789aa Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 13:52:40 -0500 Subject: [PATCH 07/21] Smoke-check the rotated token and the dispatched config post-upgrade The post-upgrade phase failed with an instant 401 from rancher.NewClient even though the rotated token had been extracted (length 66) and yq-injected into the dispatched config. Two fail-fast checks now pinpoint which side is broken: the injected token must get HTTP 200 from the upgraded Rancher API before any phase is dispatched, and the config about to be dispatched must byte-match the token in generated.tfvars. Both run under set +x so the token value stays out of the xtrace (the previous run echoed it into the build log). --- validation/Jenkinsfile.rancher-upgrade.e2e | 24 +++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 0993fde1ee..2a8f4fcf5c 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -832,10 +832,19 @@ rancher_use_bundled_system_charts: true echo "Rotated admin token extracted (length ${rotatedToken.length()}); re-injecting into test config" // The token never leaves this shell step (no Groovy/env - // interpolation), so it cannot leak into the build log. + // interpolation, set +x keeps it out of the xtrace), + // and the injected token is smoke-checked against the + // live API before any phase is dispatched on it. sh """ + set +x token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} + status=\$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) + if [ "\$status" != "200" ]; then + echo "Rotated token rejected by the upgraded Rancher API: HTTP \$status (token length \${#token})" + exit 1 + fi + echo "Rotated token smoke-check: HTTP 200 from the upgraded Rancher API" """ echo "Test config now carries the post-upgrade admin token; rancherUpgradeInput.targetVersion=${env.RANCHER_TO_VERSION}" } @@ -853,6 +862,19 @@ rancher_use_bundled_system_charts: true ).trim() def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + // Fail fast (without printing the token) if the config + // about to be dispatched does not carry the rotated one. + sh """ + set +x + expected=\$(awk -F'\"' '/^api_key/{print \$2}' ${env.INFRA_DIR}/ansible/rancher/default-ha/generated.tfvars) + actual=\$(./yq '.rancher.adminToken' ${env.TESTS_DIR}/cattle-config.yaml) + if [ "\$expected" != "\$actual" ]; then + echo "Dispatched config adminToken does not match the rotated token (expected length \${#expected}, got length \${#actual})" + exit 1 + fi + echo "Dispatched config carries the rotated admin token (length \${#expected})" + """ + echo "Dispatching post-upgrade phase: GOTEST_TESTCASE ${POST_UPGRADE_TESTCASE} (Qase run ${testRunID})" build job: params.INDIVIDUAL_JOB, parameters: [ string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), From a5420c2f09e4e63c5622301d509547466ba692d3 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 14:47:21 -0500 Subject: [PATCH 08/21] Accept the insecure LB cert in the rotated-token smoke check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The smoke-check curl died with exit 60 (TLS verification) against pitupgrade.qa.rancher.space before it ever reached auth — the same endpoint the tests already call with insecure: true. Pass -k so the check actually adjudicates the token. --- validation/Jenkinsfile.rancher-upgrade.e2e | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 2a8f4fcf5c..cb0921dd84 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -839,7 +839,7 @@ rancher_use_bundled_system_charts: true set +x token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} - status=\$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) + status=\$(curl -sk -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) if [ "\$status" != "200" ]; then echo "Rotated token rejected by the upgraded Rancher API: HTTP \$status (token length \${#token})" exit 1 From ae95642df430129804c4cb315b2998686efa448c Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 15:38:02 -0500 Subject: [PATCH 09/21] Export the token before yq injection (empty-adminToken 401 root cause) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit token=$(awk ...) creates an unexported shell variable, but yq's strenv() reads the process environment, so the injection silently wrote an empty string into rancher.adminToken — the post-upgrade phase then authenticated with an empty bearer token (instant 401). curl in the same step still worked because its header uses shell expansion, which is why the smoke-check passed while the config check failed (expected length 66, got length 0). export the variable so the child yq process can see it. --- validation/Jenkinsfile.rancher-upgrade.e2e | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index cb0921dd84..9b6e5f4d9d 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -838,6 +838,9 @@ rancher_use_bundled_system_charts: true sh """ set +x token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) + # export: yq's strenv() reads the environment, not the + # shell's unexported variables (empty-token 401 root cause). + export token ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} status=\$(curl -sk -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) if [ "\$status" != "200" ]; then From 739d5acd258790135813d988daa9f83f3618dd0e Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 17:26:09 -0500 Subject: [PATCH 10/21] Address PR review: phase progress logs, build-pinned image tags - logrus milestone logs in each upgrade phase (version resolution, chart install/upgrade, workload seed, checksum verification) so the job log shows where each phase is without reading the assertions backwards. - rancher-infra-tools/rancher-go-test now carry a build-scoped tag (pitupgrade-) used by every direct docker invocation in this pipeline, so this job no longer depends on whatever another job last built or removed under the shared :latest tag on the same agent; the :latest alias is still applied for qa-jenkins-library's make.runTarget image resolution. --- validation/Jenkinsfile.rancher-upgrade.e2e | 33 ++++++++++++++-------- validation/rancherupgrade/longhorn_test.go | 13 +++++++++ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 9b6e5f4d9d..90be3147cc 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -33,8 +33,8 @@ * infrastructure.writeSshKey, infrastructure.detectPublicIp * property.useWithProperties * - * Admin token generation uses a direct ansible-playbook call inside - * rancher-go-test:latest (no make token target exists upstream yet). + * Admin token generation uses a direct ansible-playbook call inside the + * rancher-go-test image (no make token target exists upstream yet). */ def libraryBranch = env.QA_JENKINS_LIBRARY_BRANCH ?: 'main' @@ -42,6 +42,14 @@ library "qa-jenkins-library@${libraryBranch}" def DEFAULT_RANCHER_PASSWORD = 'X%S3bPyg@n8kWcRs675l#fH0' +// Build-scoped image tags: every direct docker invocation below runs the +// pinned tag so this job never depends on (or mutates) whatever some other +// job last built under the shared :latest tag on the same agent. The :latest +// alias is still applied at build time because qa-jenkins-library's +// make.runTarget resolves that name for its own container runs. +def INFRA_TOOLS_IMAGE = "rancher-infra-tools:pitupgrade-${env.BUILD_NUMBER}" +def GO_TEST_IMAGE = "rancher-go-test:pitupgrade-${env.BUILD_NUMBER}" + // Suite/method selectors passed as GOTEST_TESTCASE to the downstream executor. // Each phase is its own go test invocation (-run regex), so cross-phase state // lives in-cluster under deterministic names, never in process memory. @@ -116,7 +124,7 @@ def collectUpgradeDiagnostics() { -v ${workspace}:/workspace \ -e KUBECONFIG=/workspace/${env.INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml \ -w /workspace \ - rancher-infra-tools:latest \ + ${INFRA_TOOLS_IMAGE} \ sh -c ' set +e kubectl get nodes -o wide > diagnostics-nodes.txt 2>&1 @@ -387,10 +395,13 @@ rancherUpgradeInput: stage('Build Docker Images') { steps { - // rancher-infra-tools:latest — used by make.groovy / infra.* for all make targets - sh "docker build --no-cache --platform linux/amd64 -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." - // rancher-go-test:latest — used for qase binary build and admin-token playbook - sh "docker build --no-cache --platform linux/amd64 -t rancher-go-test:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.airgap-go-tests ." + // rancher-infra-tools — used by make.groovy / infra.* for all make + // targets (library resolves the :latest alias); direct invocations + // in this file use the build-pinned tag. + sh "docker build --no-cache --platform linux/amd64 -t ${INFRA_TOOLS_IMAGE} -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." + // rancher-go-test — used for the qase binary build, token playbook, + // and redaction step, always via the build-pinned tag. + sh "docker build --no-cache --platform linux/amd64 -t ${GO_TEST_IMAGE} -t rancher-go-test:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.airgap-go-tests ." } } @@ -693,7 +704,7 @@ rancher_use_bundled_system_charts: true -e ANSIBLE_CONFIG=/workspace/${env.INFRA_DIR}/ansible/ansible.cfg \ -v ${workspace}:/workspace \ -w /workspace/${env.INFRA_DIR}/ansible/rke2/default \ - rancher-go-test:latest \ + ${GO_TEST_IMAGE} \ ansible-playbook -i inventory/inventory.yml /workspace/${env.INFRA_DIR}/ansible/rancher/token/generate-admin-token.yml \ -e rancher_cattle_config_file=${containerConfigPath} \ -e rancher_token_ttl=${tokenTtl} \ @@ -710,7 +721,7 @@ rancher_use_bundled_system_charts: true // Redact secret fields from token JSON via Docker container. // Pipeline Utility Steps plugin (readJSON/writeJSON) is not installed, // and python3 is not available on the Jenkins agent host. - sh "docker run --rm -v \$(pwd):/workspace -w /workspace rancher-go-test:latest python3 -c \"import json; d=json.load(open('rancher-token.json')); [d.pop(k,None) for k in ('token','bearerToken','accessKey','secretKey','value')]; json.dump(d,open('rancher-token-metadata.json','w'),indent=2)\"" + sh "docker run --rm -v \$(pwd):/workspace -w /workspace ${GO_TEST_IMAGE} python3 -c \"import json; d=json.load(open('rancher-token.json')); [d.pop(k,None) for k in ('token','bearerToken','accessKey','secretKey','value')]; json.dump(d,open('rancher-token-metadata.json','w'),indent=2)\"" if (params.REPORT_ARTIFACTS) { archiveArtifacts artifacts: 'rancher-token-metadata.json' echo 'Archived redacted token metadata: rancher-token-metadata.json' @@ -750,7 +761,7 @@ rancher_use_bundled_system_charts: true docker run --rm --platform linux/amd64 \ -v ${workspace}:/workspace \ -w /workspace/${env.TESTS_DIR} \ - rancher-go-test:latest \ + ${GO_TEST_IMAGE} \ sh -c 'env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /workspace/testrun ./validation/pipeline/qase/testrun' """ @@ -971,7 +982,7 @@ rancher_use_bundled_system_charts: true } // Docker cleanup - sh 'docker rmi -f rancher-go-test:latest rancher-infra-tools:latest || true' + sh "docker rmi -f ${GO_TEST_IMAGE} ${INFRA_TOOLS_IMAGE} rancher-go-test:latest rancher-infra-tools:latest || true" // Sensitive artifact cleanup — shred or remove credentials from workspace sh ''' diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go index 379fb38b9e..bb4b77a54c 100644 --- a/validation/rancherupgrade/longhorn_test.go +++ b/validation/rancherupgrade/longhorn_test.go @@ -16,6 +16,7 @@ import ( actionsClusters "github.com/rancher/tests/actions/clusters" "github.com/rancher/tests/actions/projects" "github.com/rancher/tests/actions/storage" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -65,6 +66,7 @@ func (s *LonghornUpgradeTestSuite) TearDownSuite() { } func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { + logrus.Infof("pre-upgrade phase: creating project on cluster %s", s.cluster.ID) project, _, err := projects.CreateProjectAndNamespace(s.client, s.cluster.ID) require.NoError(s.T(), err) @@ -77,6 +79,7 @@ func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { require.NoError(s.T(), err) version, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: installing longhorn %s from the %s repo", version, catalog.RancherChartRepo) err = charts.InstallLonghornChart(s.client, charts.PayloadOpts{ Namespace: charts.LonghornNamespace, @@ -88,29 +91,36 @@ func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { }, }, nil) require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: longhorn %s installed; seeding checksum workload %s", version, checksumWorkloadName) err = storage.CreateChecksumWorkload(s.client, s.cluster.ID, charts.LonghornStorageClass, checksumWorkloadName, checksumContent) require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: checksum workload %s ready; pre-upgrade state seeded", checksumWorkloadName) } func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { cfg := loadUpgradeInput() require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") + logrus.Infof("post-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: server version %s observed; checking longhorn survival", cfg.TargetVersion) chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) require.NoError(s.T(), err) require.True(s.T(), chart.IsAlreadyInstalled, "longhorn chart must still be installed after the Rancher server upgrade") + logrus.Infof("post-upgrade phase: longhorn chart still installed; verifying workload data") err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: checksum verified on %s", checksumWorkloadName) } func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { cfg := loadUpgradeInput() require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") + logrus.Infof("chart-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) require.NoError(s.T(), err) @@ -122,6 +132,7 @@ func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { require.NoError(s.T(), err) latest, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) require.NoError(s.T(), err) + logrus.Infof("chart-upgrade phase: upgrading longhorn to %s from the upgraded catalog", latest) installOptions := &charts.InstallOptions{ Cluster: s.cluster, @@ -135,9 +146,11 @@ func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) require.NoError(s.T(), err) require.Equal(s.T(), latest, chart.ChartDetails.Spec.Chart.Metadata.Version) + logrus.Infof("chart-upgrade phase: longhorn at %s; verifying workload data", latest) err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) require.NoError(s.T(), err) + logrus.Infof("chart-upgrade phase: checksum verified on %s", checksumWorkloadName) } func TestLonghornUpgradeTestSuite(t *testing.T) { From b14003d896360936c8199a8d0c0c856612569ca4 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Mon, 19 Oct 2026 08:23:54 -0500 Subject: [PATCH 11/21] fix: decide teardown in post.always, defer artifact removal to cleanup Jenkins evaluates post conditions in a fixed order (always before failure, cleanup last), so the failure{} destroy block could never work: always had already shredded workspace_name.txt and tfvars and removed the infra-tools image teardown depends on, and with DESTROY_AFTER_TESTS=false + DESTROY_ON_FAILURE=true a failed run skipped teardown entirely, leaking the AWS environment. Fold the destroy decision into always (after diagnostics and Qase completion), gated on DESTROY_AFTER_TESTS or failure+DESTROY_ON_ FAILURE; move the docker rmi and sensitive-artifact shredding into a cleanup block so they run last. --- validation/Jenkinsfile.rancher-upgrade.e2e | 37 ++++++++++++---------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 90be3147cc..b53e126f2c 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -947,21 +947,13 @@ rancher_use_bundled_system_charts: true } post { - failure { - script { - if (params.DESTROY_ON_FAILURE) { - destroyInfrastructure('DESTROY_ON_FAILURE') - } - } - } - always { script { - // Failure diagnostics run here, not in the failure block: - // Jenkins evaluates post conditions in a fixed order with - // always first, so anything in failure{} would already have - // lost the docker images (rmi below) and the live cluster - // (destroy below). Collect while both still exist. + // Failure diagnostics run first, while the live cluster and + // the docker images still exist: Jenkins evaluates post + // conditions in a fixed order (always before failure, cleanup + // last), so every failure-path action must run in this block, + // ahead of the teardown below. if (currentBuild.currentResult != 'SUCCESS') { collectUpgradeDiagnostics() } @@ -977,11 +969,24 @@ rancher_use_bundled_system_charts: true } } - if (params.DESTROY_AFTER_TESTS) { - destroyInfrastructure('DESTROY_AFTER_TESTS') + // Single teardown decision point. The failure-path destroy is + // decided here rather than in a failure{} block because always + // runs first: destroyInfrastructure needs workspace_name.txt, + // the tfvars and the infra-tools image, which are only removed + // afterwards in cleanup{}. DESTROY_ON_FAILURE=false on a failed + // run keeps the environment around for debugging. + def destroyForFailure = currentBuild.currentResult != 'SUCCESS' && params.DESTROY_ON_FAILURE + if (params.DESTROY_AFTER_TESTS || destroyForFailure) { + destroyInfrastructure(params.DESTROY_AFTER_TESTS ? 'DESTROY_AFTER_TESTS' : 'DESTROY_ON_FAILURE') } - // Docker cleanup + } + } + + cleanup { + script { + // Docker cleanup — cleanup{} runs after always, so the + // teardown above can still resolve the infra-tools image. sh "docker rmi -f ${GO_TEST_IMAGE} ${INFRA_TOOLS_IMAGE} rancher-go-test:latest rancher-infra-tools:latest || true" // Sensitive artifact cleanup — shred or remove credentials from workspace From 84bac3d892463a06185882f846c08517db24dbda Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 20 Oct 2026 10:36:39 -0500 Subject: [PATCH 12/21] Address PR review: pit.weekly build tag, skip suite outside the upgrade pipeline --- validation/rancherupgrade/README.md | 6 +++++- validation/rancherupgrade/longhorn_test.go | 10 +++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md index 7436559c6b..6ef4686615 100644 --- a/validation/rancherupgrade/README.md +++ b/validation/rancherupgrade/README.md @@ -23,6 +23,10 @@ Do not "fix" the following; both are required by the upgrade workflow (rancher/t the cleanup. - The suite is admin-only: upgrade phases assert server-level state and the pipeline runs them with the admin token. +- The build tag is `(validation || pit.weekly)`: generic scheduled PIT legs deploy + Rancher once and never upgrade it, so `SetupSuite` skips the suite when + `rancherUpgradeInput.targetVersion` is unset; only the dedicated rancher-upgrade + pipeline sets it. ## Config @@ -38,7 +42,7 @@ rancherUpgradeInput: ``` `rancher.clusterName` selects the shared downstream cluster. `rancherUpgradeInput.targetVersion` -is required by the post-upgrade and chart-upgrade phases; the pipeline writes it from +is required by every phase (the suite skips without it); the pipeline writes it from the qa-infra-automation resolver output, or set it by hand for local runs. ## Running diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go index bb4b77a54c..e334d86f46 100644 --- a/validation/rancherupgrade/longhorn_test.go +++ b/validation/rancherupgrade/longhorn_test.go @@ -1,4 +1,4 @@ -//go:build validation && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 +//go:build (validation || pit.weekly) && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 package rancherupgrade @@ -47,6 +47,12 @@ func loadUpgradeInput() *UpgradeInput { } func (s *LonghornUpgradeTestSuite) SetupSuite() { + // Generic PIT legs never upgrade the server between phases; without an + // upgrade target the phases have nothing to assert, so skip the suite. + if loadUpgradeInput().TargetVersion == "" { + s.T().Skipf("%s.targetVersion is not set: this suite runs only in the rancher-upgrade pipeline, which upgrades the server between phases", UpgradeInputConfigKey) + } + testSession := session.NewSession() s.session = testSession @@ -100,7 +106,6 @@ func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { cfg := loadUpgradeInput() - require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") logrus.Infof("post-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) @@ -119,7 +124,6 @@ func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { cfg := loadUpgradeInput() - require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") logrus.Infof("chart-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) From 08487294e57028a1a58f963dd0d6bd3d6358577d Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 20 Oct 2026 10:46:07 -0500 Subject: [PATCH 13/21] Address PR review: fail on pipeline misconfig via rancherUpgradePipeline marker --- validation/Jenkinsfile.rancher-upgrade.e2e | 6 +++++- validation/rancherupgrade/README.md | 5 +++-- validation/rancherupgrade/longhorn_test.go | 13 ++++++++++--- validation/rancherupgrade/rancherupgrade.go | 11 +++++++++++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index b53e126f2c..29a01e7f39 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -330,7 +330,11 @@ rancher: # ("local") cluster; the upgrade suite installs Longhorn on it. clusterName: local rancherUpgradeInput: - targetVersion: "${RANCHER_VERSION_UPGRADE}"''', + targetVersion: "${RANCHER_VERSION_UPGRADE}" +# Marker: set only by this pipeline. The suite fails rather than skips when +# this marker is present but the target version is missing (pipeline bug). +rancherUpgradePipeline: + enabled: true''', ) booleanParam( name: 'REPORT_ARTIFACTS', diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md index 6ef4686615..0197f60c62 100644 --- a/validation/rancherupgrade/README.md +++ b/validation/rancherupgrade/README.md @@ -25,8 +25,9 @@ Do not "fix" the following; both are required by the upgrade workflow (rancher/t runs them with the admin token. - The build tag is `(validation || pit.weekly)`: generic scheduled PIT legs deploy Rancher once and never upgrade it, so `SetupSuite` skips the suite when - `rancherUpgradeInput.targetVersion` is unset; only the dedicated rancher-upgrade - pipeline sets it. + `rancherUpgradeInput.targetVersion` is unset. The dedicated pipeline marks its + config with `rancherUpgradePipeline.enabled: true`; with the marker present but + the target missing, the suite fails (pipeline bug) instead of skipping. ## Config diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go index e334d86f46..f1b5f5d7bb 100644 --- a/validation/rancherupgrade/longhorn_test.go +++ b/validation/rancherupgrade/longhorn_test.go @@ -47,9 +47,16 @@ func loadUpgradeInput() *UpgradeInput { } func (s *LonghornUpgradeTestSuite) SetupSuite() { - // Generic PIT legs never upgrade the server between phases; without an - // upgrade target the phases have nothing to assert, so skip the suite. - if loadUpgradeInput().TargetVersion == "" { + cfg := loadUpgradeInput() + pipeline := new(PipelineInput) + config.LoadConfig(PipelineConfigKey, pipeline) + + if cfg.TargetVersion == "" { + // The dedicated pipeline always writes both keys, so an empty target + // there is a pipeline bug and must fail, not skip. + require.False(s.T(), pipeline.Enabled, "rancher-upgrade pipeline config is missing %s.targetVersion", UpgradeInputConfigKey) + // Generic PIT legs never upgrade the server between phases; without + // an upgrade target the phases have nothing to assert, so skip. s.T().Skipf("%s.targetVersion is not set: this suite runs only in the rancher-upgrade pipeline, which upgrades the server between phases", UpgradeInputConfigKey) } diff --git a/validation/rancherupgrade/rancherupgrade.go b/validation/rancherupgrade/rancherupgrade.go index a08cf104eb..5d3aeb112a 100644 --- a/validation/rancherupgrade/rancherupgrade.go +++ b/validation/rancherupgrade/rancherupgrade.go @@ -8,6 +8,10 @@ package rancherupgrade const ( // UpgradeInputConfigKey is the cattle-config key holding upgrade-phase inputs. UpgradeInputConfigKey = "rancherUpgradeInput" + + // PipelineConfigKey marks a config as authored by the rancher-upgrade + // pipeline; generic PIT legs never set it. + PipelineConfigKey = "rancherUpgradePipeline" ) // UpgradeInput holds per-run upgrade parameters. TargetVersion is the Rancher @@ -16,3 +20,10 @@ const ( type UpgradeInput struct { TargetVersion string `json:"targetVersion" yaml:"targetVersion"` } + +// PipelineInput is the marker section the rancher-upgrade pipeline writes into +// its cattle-config; the suite fails rather than skips when it is present but +// the upgrade target is missing (pipeline bug, not a generic PIT leg). +type PipelineInput struct { + Enabled bool `json:"enabled" yaml:"enabled"` +} From da6002853f2e9f14ab3e1425ee87bc1bcc6afc30 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 21 Oct 2026 14:33:43 -0500 Subject: [PATCH 14/21] Address PR 986 review findings: checksum validation, stale catalog guard, flavor plumbing - actions/storage: reject checksum content outside [A-Za-z0-9-] before sh -c interpolation - rancherupgrade: fail fast when resolved latest equals the installed chart version (stale cluster-scoped catalog follower) - Jenkinsfile.rancher-upgrade.e2e: document timeout budget, plumb RANCHER_CHART_REPO_FLAVOR into make args, resolve kubeconfig via INFRA_DIR, select deploy repo by flavor - README + pit_schemas: correct downstream-cluster wording to the Rancher local cluster - actions/clusters: debug logging on server-version read/parse failures --- actions/clusters/local.go | 4 ++ actions/storage/checksum.go | 12 +++++ validation/Jenkinsfile.rancher-upgrade.e2e | 48 +++++++++++++------ validation/rancherupgrade/README.md | 29 ++++++----- validation/rancherupgrade/longhorn_test.go | 15 +++++- .../rancherupgrade/schemas/pit_schemas.yaml | 6 +-- 6 files changed, 82 insertions(+), 32 deletions(-) diff --git a/actions/clusters/local.go b/actions/clusters/local.go index 752df582ee..16940dfd53 100644 --- a/actions/clusters/local.go +++ b/actions/clusters/local.go @@ -56,11 +56,15 @@ func WaitRancherVersion(client *rancher.Client, targetVersion string, timeout ti serverVersionRawValue, err := client.Management.Setting.ByID(serverVersionSetting) if err != nil { // The server may be mid-restart during the upgrade rollout; retry the read. + // Logged so a terminal cause (e.g. a revoked token or unreachable API) + // is visible in the log instead of surfacing only as the final timeout. + logrus.Debugf("server-version read failed, retrying: %v", err) return false, nil } serverVersion, err := parseRancherVersion(serverVersionRawValue.Value) if err != nil { + logrus.Debugf("server-version value %q not parseable yet, retrying: %v", serverVersionRawValue.Value, err) return false, nil } diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go index 9f3d173cf7..6c5dcf2e59 100644 --- a/actions/storage/checksum.go +++ b/actions/storage/checksum.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "fmt" + "regexp" "strings" "github.com/rancher/shepherd/clients/rancher" @@ -26,6 +27,12 @@ import ( // ChecksumDataFile is the filename written under MountPath by CreateChecksumWorkload. const ChecksumDataFile = "checksum-data" +// checksumContentPattern constrains the content accepted by CreateChecksumWorkload: +// content is interpolated into a single-quoted sh -c command, so a value outside +// alphanumeric/hyphen (notably a quote) would break the quoting. Enforced in code +// so a future caller cannot silently inject shell. +var checksumContentPattern = regexp.MustCompile(`^[A-Za-z0-9-]+$`) + // CreateChecksumWorkload creates a 1GiB PVC named on storageClassName in the // default namespace, mounts it into a single-replica nginx deployment named at // MountPath, waits for rollout, then writes content to MountPath/ChecksumDataFile via @@ -34,6 +41,11 @@ const ChecksumDataFile = "checksum-data" // it is interpolated into an sh -c command. No session cleanup is registered: callers // are upgrade tests whose resources must outlive the process. func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageClassName string, name string, content string) error { + if !checksumContentPattern.MatchString(content) { + return fmt.Errorf("content %q must match %s: it is interpolated into a single-quoted sh -c command", + content, checksumContentPattern.String()) + } + storageClass, err := GetStorageClass(client, clusterID, storageClassName) if err != nil { return err diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 29a01e7f39..7b3b6ac24d 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -148,8 +148,11 @@ pipeline { options { ansiColor('xterm') - // Deploy + three sequential test phases + server upgrade + destroy - timeout(time: 240, unit: 'MINUTES') + // Budget: three sequential test phases (TEST_TIMEOUT each, default 60m) + // plus deploy, server upgrade, and destroy (~90m overhead). Must exceed + // the sum of the stage budgets or a legitimately slow run aborts + // mid-phase; revisit when raising TEST_TIMEOUT. + timeout(time: 300, unit: 'MINUTES') buildDiscarder(logRotator(numToKeepStr: '30')) } @@ -200,7 +203,7 @@ pipeline { string( name: 'RANCHER_CHART_REPO_FLAVOR', defaultValue: 'community', - description: 'Chart repo flavor for the upgrade (community|prime; prime is a later slice)' + description: 'Chart repo flavor (community|prime): selects the chart index for version resolution, the repo the pre-upgrade Rancher is deployed from, and the repo the server upgrade pulls from' ) // ── Infrastructure parameters ────────────────────────── @@ -232,7 +235,7 @@ pipeline { booleanParam( name: 'DESTROY_AFTER_TESTS', defaultValue: true, - description: 'Tear down infrastructure after tests complete' + description: 'Tear down infrastructure after a successful run; failed runs are governed by DESTROY_ON_FAILURE' ) // ── Terraform parameters ─────────────────────────────── @@ -288,7 +291,7 @@ nodes = [ defaultValue: '''--- kubernetes_version: ${RKE2_VERSION} cni: calico -kubeconfig_file: "/workspace/qa-infra-automation/ansible/rke2/default/kubeconfig.yaml" +kubeconfig_file: "/workspace/${INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml" cert_manager_version: "1.20.0" bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" password: "${RANCHER_ADMIN_PASSWORD}" @@ -449,7 +452,10 @@ rancherUpgradePipeline: def resolveOut = make.runTarget( target: 'rancher-resolve-version', dir: env.INFRA_DIR, - makeArgs: ['SOURCE_LINE=2.14'], + makeArgs: [ + 'SOURCE_LINE=2.14', + "RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}" + ], returnStdout: true, passAwsCreds: false ) @@ -472,6 +478,7 @@ rancherUpgradePipeline: def resolveOut = make.runTarget( target: 'rancher-resolve-version', dir: env.INFRA_DIR, + makeArgs: ["RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}"], returnStdout: true, passAwsCreds: false ) @@ -592,15 +599,19 @@ rancherUpgradePipeline: stage('Configure Ansible Variables') { sh "mkdir -p ${ansiblePath}/inventory/group_vars" - // The FROM version is a stable semver patch line, so - // its chart repo is the stable "latest" index. - def fromChartRepoUrl = 'https://releases.rancher.com/server-charts/latest' + // Flavor selects the deploy-side repo too: the prime + // leg installs the FROM version from the prime index + // (community stops the 2.14 line at 2.14.3 while prime + // continues it), keeping resolve, deploy, and upgrade + // all on the same artifact line. + def fromChartRepoUrl = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'https://charts.rancher.com/server-charts/prime' : 'https://releases.rancher.com/server-charts/latest' def upgradeRepoName = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'rancher-prime-upgrade' : 'rancher-upgrade' def ansibleVars = infrastructure.parseAndSubstituteVars( content: params.ANSIBLE_VARIABLES, envVars: [ 'RKE2_VERSION': params.RKE2_VERSION, + 'INFRA_DIR': env.INFRA_DIR, 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, 'RANCHER_VERSION': env.RANCHER_FROM_VERSION, @@ -977,11 +988,15 @@ rancher_use_bundled_system_charts: true // decided here rather than in a failure{} block because always // runs first: destroyInfrastructure needs workspace_name.txt, // the tfvars and the infra-tools image, which are only removed - // afterwards in cleanup{}. DESTROY_ON_FAILURE=false on a failed - // run keeps the environment around for debugging. - def destroyForFailure = currentBuild.currentResult != 'SUCCESS' && params.DESTROY_ON_FAILURE - if (params.DESTROY_AFTER_TESTS || destroyForFailure) { - destroyInfrastructure(params.DESTROY_AFTER_TESTS ? 'DESTROY_AFTER_TESTS' : 'DESTROY_ON_FAILURE') + // afterwards in cleanup{}. A failed run is governed by + // DESTROY_ON_FAILURE alone and a successful one by + // DESTROY_AFTER_TESTS, so DESTROY_ON_FAILURE=false on a failed + // run keeps the environment around for debugging even when + // DESTROY_AFTER_TESTS is true. + def buildSucceeded = currentBuild.currentResult == 'SUCCESS' + def shouldDestroy = buildSucceeded ? params.DESTROY_AFTER_TESTS : params.DESTROY_ON_FAILURE + if (shouldDestroy) { + destroyInfrastructure(buildSucceeded ? 'DESTROY_AFTER_TESTS' : 'DESTROY_ON_FAILURE') } } @@ -1028,9 +1043,12 @@ rancher_use_bundled_system_charts: true cleanup_file "workspace_name.txt" cleanup_file "resolve-from.json" cleanup_file "resolve-to.json" + # cattle-config carries the admin password and the live rotated + # admin token; group_vars/all.yml carries the rancher passwords. + cleanup_file "${TESTS_DIR}/cattle-config.yaml" if [ -n "${INFRA_DIR}" ] && [ -d "${INFRA_DIR}" ]; then - find "${INFRA_DIR}" -type f \\( -name 'terraform.tfvars' -o -name 'generated.tfvars' -o -name '*.pem' -o -name 'id_rsa' -o -name 'id_ed25519' \\) -exec sh -c ' + find "${INFRA_DIR}" -type f \\( -name 'terraform.tfvars' -o -name 'generated.tfvars' -o -name 'all.yml' -o -name '*.pem' -o -name 'id_rsa' -o -name 'id_ed25519' \\) -exec sh -c ' for file do if command -v shred >/dev/null 2>&1; then shred -u "$file" || rm -f "$file" diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md index 0197f60c62..c8161aea50 100644 --- a/validation/rancherupgrade/README.md +++ b/validation/rancherupgrade/README.md @@ -36,15 +36,17 @@ rancher: host: adminToken: insecure: true - clusterName: "" # pre-provisioned by qa-infra-automation + clusterName: "local" # Rancher's own local cluster: the qa-infra-automation RKE2 HA deploy rancherUpgradeInput: targetVersion: "" ``` - -`rancher.clusterName` selects the shared downstream cluster. `rancherUpgradeInput.targetVersion` -is required by every phase (the suite skips without it); the pipeline writes it from -the qa-infra-automation resolver output, or set it by hand for local runs. +`rancher.clusterName` selects the target cluster. The pipeline uses `local` — Rancher's own +3-node HA RKE2 cluster provisioned by qa-infra-automation — so Longhorn and its volume are +exercised on the server cluster itself, matching what the pipeline deploys. +`rancherUpgradeInput.targetVersion` is required by every phase (the suite skips without it); +the pipeline writes it from the qa-infra-automation resolver output, or set it by hand for +local runs. ## Running @@ -70,9 +72,9 @@ gotestsum --format standard-verbose --packages=github.com/rancher/tests/validati ## Notes -- The shared downstream RKE2 cluster is provisioned by qa-infra-automation and must be - sized for Longhorn plus the later NeuVector/Observability suites; this is an infra - concern and is not configurable here. +- The target cluster is Rancher's own local cluster (the qa-infra-automation 3-node HA + RKE2 deploy), so the server nodes themselves must be sized for Longhorn plus the later + NeuVector/Observability suites; this is an infra concern and is not configurable here. - The Rancher server upgrade itself is `make rancher-upgrade` in [qa-infra-automation](https://github.com/rancher/qa-infra-automation); this package only runs the pre- and post-upgrade assertions around it. @@ -80,8 +82,9 @@ gotestsum --format standard-verbose --packages=github.com/rancher/tests/validati `ansible/rancher/-ha/generated.tfvars`; the pipeline must copy it into the test config's `rancher.adminToken` before the post-upgrade phases (the pre-upgrade token is revoked by the upgrade). -- After a server upgrade the downstream cluster-scoped catalog follower can keep - serving the previous Rancher line's index (e.g. release-v2.14 after upgrading to - 2.15) until the downstream `cattle-cluster-agent` re-syncs. If the chart-upgrade - phase fails with "failed to find chart longhorn from the chart repo", restart the - downstream cluster-agent and retry; observed re-sync time ~4 minutes. +- After a server upgrade the cluster-scoped catalog follower can keep serving the + previous Rancher line's index (e.g. release-v2.14 after upgrading to 2.15) until the + `cattle-cluster-agent` re-syncs. The chart-upgrade phase detects this and fails with + an explicit stale-catalog message when the catalog's latest equals the installed + version, or with "failed to find chart longhorn from the chart repo" on a lookup + miss; restart the cluster agent and re-run the phase. Observed re-sync time ~4 minutes. diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go index f1b5f5d7bb..3b203420d6 100644 --- a/validation/rancherupgrade/longhorn_test.go +++ b/validation/rancherupgrade/longhorn_test.go @@ -143,7 +143,20 @@ func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { require.NoError(s.T(), err) latest, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) require.NoError(s.T(), err) - logrus.Infof("chart-upgrade phase: upgrading longhorn to %s from the upgraded catalog", latest) + + // Guard against a stale cluster-scoped catalog: until the cluster agent + // re-syncs after the server upgrade, the follower can still serve the + // previous Rancher line's index, whose latest equals the installed version. + // Upgrading to it would pass vacuously (a same-version upgrade asserted + // equal to itself), so fail with the runbook remedy instead; see README Notes. + installed, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.NotEqual(s.T(), + installed.ChartDetails.Spec.Chart.Metadata.Version, latest, + "cluster-scoped catalog still serves the pre-upgrade index: installed longhorn %s equals latest %s; restart the cluster agent and re-run this phase", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + logrus.Infof("chart-upgrade phase: upgrading longhorn %s to %s from the upgraded catalog", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) installOptions := &charts.InstallOptions{ Cluster: s.cluster, diff --git a/validation/rancherupgrade/schemas/pit_schemas.yaml b/validation/rancherupgrade/schemas/pit_schemas.yaml index da764c080d..a95dc71f59 100644 --- a/validation/rancherupgrade/schemas/pit_schemas.yaml +++ b/validation/rancherupgrade/schemas/pit_schemas.yaml @@ -5,7 +5,7 @@ description: "Install Longhorn and seed a checksum workload on a Longhorn volume before the Rancher server upgrade" automation: 2 steps: - - action: "Create a project on the downstream cluster" + - action: "Create a project on the Rancher local cluster" data: "" expectedresult: "The project is created successfully" position: 1 @@ -47,11 +47,11 @@ position: 1 - action: "Resolve the latest Longhorn chart version from the cluster-scoped catalog" data: "Chart: longhorn from the rancher-charts repo" - expectedresult: "The latest cluster-scoped chart version is resolved" + expectedresult: "The latest cluster-scoped chart version is resolved and differs from the installed pre-upgrade version" position: 2 - action: "Upgrade the Longhorn chart with retry" data: "longhorn and longhorn-crd to the latest version" - expectedresult: "Both charts reach Deployed state and workloads become Ready" + expectedresult: "The longhorn chart reaches Deployed state and workloads become Ready" position: 3 - action: "Verify the chart version and the checksum workload" data: "Chart version equals latest; PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" From a19485d98e37dab2dd3494359fb6faf48802cc9d Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 21 Oct 2026 14:38:49 -0500 Subject: [PATCH 15/21] Address hamistao review findings: dedupe deployment helper, clarify version scope - pvc.go: createNginxDeploymentWithPVC takes an explicit deploymentName, so CreatePVCWorkload keeps random names while CreateChecksumWorkload passes its deterministic one; shared construction replaces ~25 duplicated lines - checksum.go: reuse the shared helper; drop now-unused charts/wloads imports - README: previous-release-line-to-latest wording (pipeline default, not a suite constraint) and a deliberate-deviation bullet for the out-of-band server upgrade --- actions/storage/checksum.go | 38 ++++------------------------- actions/storage/pvc.go | 23 ++++++++++------- validation/rancherupgrade/README.md | 15 ++++++++++-- 3 files changed, 32 insertions(+), 44 deletions(-) diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go index 6c5dcf2e59..e7da7d5b29 100644 --- a/actions/storage/checksum.go +++ b/actions/storage/checksum.go @@ -9,13 +9,11 @@ import ( "github.com/rancher/shepherd/clients/rancher" steveV1 "github.com/rancher/shepherd/clients/rancher/v1" - "github.com/rancher/shepherd/extensions/charts" "github.com/rancher/shepherd/extensions/defaults" "github.com/rancher/shepherd/extensions/defaults/namespaces" "github.com/rancher/shepherd/extensions/defaults/stevetypes" "github.com/rancher/shepherd/extensions/kubeapi/cluster" "github.com/rancher/shepherd/extensions/kubeconfig" - wloads "github.com/rancher/shepherd/extensions/workloads" "github.com/rancher/tests/actions/kubeapi/volumes/persistentvolumeclaims" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" @@ -94,37 +92,11 @@ func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageCla } logrus.Debugf("PVC %s is Bound to volume %s", pvc.Name, pvc.Spec.VolumeName) - volMount := corev1.VolumeMount{ - MountPath: MountPath, - Name: name, - } - - podVol := corev1.Volume{ - Name: name, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: name, - }, - }, - } - - steveclient, err := client.Steve.ProxyDownstream(clusterID) - if err != nil { - return err - } - - containerTemplate := wloads.NewContainer(nginxName, nginxName, corev1.PullAlways, []corev1.VolumeMount{volMount}, []corev1.EnvFromSource{}, nil, nil, nil) - podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: name}, nil) - deployment := wloads.NewDeploymentTemplate(name, namespaces.Default, podTemplate, true, nil) - - _, err = steveclient.SteveType(stevetypes.Deployment).Create(deployment) - if err != nil { - return err - } - - err = charts.WatchAndWaitDeployments(client, clusterID, namespaces.Default, metav1.ListOptions{ - FieldSelector: "metadata.name=" + name, - }) + // Shared nginx+PVC deployment construction with CreatePVCWorkload; the + // deterministic name makes PVC, volume mount, deployment, and relocation + // label all equal name, so later phases re-find the workload from a + // separate process. + _, err = createNginxDeploymentWithPVC(client, clusterID, name, name, name) if err != nil { return err } diff --git a/actions/storage/pvc.go b/actions/storage/pvc.go index 163c0df025..4030e6823d 100644 --- a/actions/storage/pvc.go +++ b/actions/storage/pvc.go @@ -158,7 +158,6 @@ func CreatePVC(client *rancher.Client, clusterID string, storageClassName string } return false, err }) - if err != nil { return nil, "", err } @@ -185,19 +184,25 @@ func CreatePVCWorkload(t *testing.T, client *rancher.Client, clusterID string, s persistentVolumeClaim, volumeName, err := CreatePVC(client, clusterID, storageClassName) require.NoError(t, err) - logrus.Debugf("Creating Nginx deployment with %s PVC (volume name %s)", persistentVolumeClaim.Name, volumeName) - nginxResponse, err := createNginxDeploymentWithPVC(client, clusterID, persistentVolumeClaim.Name, volumeName) + // Random workload name: this helper targets single-process functional tests; + // callers needing a deterministic (cross-phase relocatable) name call + // createNginxDeploymentWithPVC directly (see CreateChecksumWorkload). + deploymentName := namegenerator.AppendRandomString(containerNamePrefix) + logrus.Debugf("Creating Nginx deployment %s with %s PVC (volume name %s)", deploymentName, persistentVolumeClaim.Name, volumeName) + nginxResponse, err := createNginxDeploymentWithPVC(client, clusterID, persistentVolumeClaim.Name, volumeName, deploymentName) require.NoError(t, err) return nginxResponse } -// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace -func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcName, volName string) (*steveV1.SteveAPIObject, error) { +// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace. +// The deployment and its DeploymentIdentifierLabel both use deploymentName, letting callers +// pick a random name (functional tests) or a deterministic name (workloads a later test +// phase must re-find from a separate process). +func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcName, volName, deploymentName string) (*steveV1.SteveAPIObject, error) { logrus.Tracef("Vol: %s", volName) logrus.Tracef("Pod: %s", pvcName) - containerName := namegenerator.AppendRandomString(containerNamePrefix) volMount := &corev1.VolumeMount{ MountPath: MountPath, Name: volName, @@ -218,8 +223,8 @@ func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcN } containerTemplate := wloads.NewContainer(nginxName, nginxName, corev1.PullAlways, []corev1.VolumeMount{*volMount}, []corev1.EnvFromSource{}, nil, nil, nil) - podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: containerName}, nil) - deployment := wloads.NewDeploymentTemplate(containerName, namespaces.Default, podTemplate, true, nil) + podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: deploymentName}, nil) + deployment := wloads.NewDeploymentTemplate(deploymentName, namespaces.Default, podTemplate, true, nil) deploymentResp, err := steveclient.SteveType(stevetypes.Deployment).Create(deployment) if err != nil { @@ -227,7 +232,7 @@ func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcN } err = charts.WatchAndWaitDeployments(client, clusterID, namespaces.Default, metav1.ListOptions{ - FieldSelector: "metadata.name=" + containerName, + FieldSelector: "metadata.name=" + deploymentName, }) return deploymentResp, err diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md index c8161aea50..901b0f5b32 100644 --- a/validation/rancherupgrade/README.md +++ b/validation/rancherupgrade/README.md @@ -1,7 +1,12 @@ # Rancher Upgrade -The `rancherupgrade` package validates upgrading the Rancher server (2.14.x -> latest) -with interoperability products installed. The first vertical is Longhorn: a pre-upgrade +The `rancherupgrade` package validates upgrading the Rancher server from the previous +release line to the latest (currently 2.14.x -> 2.15.x) with interoperability products +installed. The line pair is the pipeline's default, not a suite constraint: the suite +itself is version-agnostic and gates on `rancherUpgradeInput.targetVersion`, while the +pipeline resolves the source via `RANCHER_VERSION_FROM` (empty = latest 2.14.x patch via +`SOURCE_LINE=2.14`) and the target via `RANCHER_VERSION_TO_UPGRADE` (empty = latest +released final). The first vertical is Longhorn: a pre-upgrade phase installs the Longhorn chart and seeds a checksum workload on a Longhorn volume, the Rancher server is then upgraded out-of-band (qa-infra-automation `make rancher-upgrade`), and post-upgrade phases verify the server version, the surviving Longhorn installation, @@ -23,6 +28,12 @@ Do not "fix" the following; both are required by the upgrade workflow (rancher/t the cleanup. - The suite is admin-only: upgrade phases assert server-level state and the pipeline runs them with the admin token. +- The server upgrade stays out-of-band (a pipeline stage running qa-infra-automation + `make rancher-upgrade`) rather than a step inside the test: the go test phases + assert outcomes only and hold no infra credentials, the upgrade needs the + controller-side qa-infra checkout and its terraform/ansible credentials, and + per-phase Jenkins stage status is the acceptance signal; pulling the upgrade into + the test process would collapse the three stages into one. - The build tag is `(validation || pit.weekly)`: generic scheduled PIT legs deploy Rancher once and never upgrade it, so `SetupSuite` skips the suite when `rancherUpgradeInput.targetVersion` is unset. The dedicated pipeline marks its From 96940e5b62a14622fd52c7c7276d79768cb45279 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 22 Oct 2026 09:09:07 -0500 Subject: [PATCH 16/21] Generalize rancher-upgrade pipeline to a TEST_MODULE selector (#968) Replace the three hardcoded Longhorn -run regexes with an UPGRADE_TEST_MODULES map (longhorn, neuvector, fleet, observability) selected by a new TEST_MODULE parameter (default longhorn, preserving existing behavior). The checkout stage fails fast on an unknown module before any infra spend and logs the selected regexes per stage. Stages renamed to module-agnostic names; the chart-upgrade stage is skipped for fleet (no standalone chart), which closes the shared Qase run in the post-upgrade stage instead. Discharges the #986 review discussion about generic post-upgrade stages as modules land. --- validation/Jenkinsfile.rancher-upgrade.e2e | 97 +++++++++++++++++----- 1 file changed, 76 insertions(+), 21 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 7b3b6ac24d..3c406390c8 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -5,13 +5,14 @@ * Six-stage flow, community leg: * 1. Deploys a 2.14-latest environment via qa-infra-automation * (make backend-s3, workspace-new, infra-up, cluster, rancher) - * 2. Pre-upgrade phase: installs Longhorn + seeds a checksum workload - * (validation/rancherupgrade, TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade) + * 2. Pre-upgrade phase: installs the TEST_MODULE chart + seeds its upgrade + * state (validation/rancherupgrade, UpgradeTestSuite/*PreUpgrade) * 3. Upgrades the Rancher server in place via `make rancher-upgrade` * and re-injects the rotated admin token into the test config - * 4. Post-upgrade phase: asserts server version, Longhorn survival, workload data - * 5. Chart-upgrade phase: upgrades Longhorn to the version offered by the - * upgraded Rancher's catalog and re-verifies the workload data + * 4. Post-upgrade phase: asserts server version and module state survival + * 5. Chart-upgrade phase: upgrades the module chart to the version offered + * by the upgraded Rancher's catalog and re-verifies (skipped for fleet: + * Fleet ships inside Rancher with no standalone chart) * 6. Always destroys the environment (infra-down AUTO_APPROVE=yes) * * Versions resolve at runtime when RANCHER_VERSION_FROM / RANCHER_VERSION_TO_UPGRADE @@ -53,9 +54,34 @@ def GO_TEST_IMAGE = "rancher-go-test:pitupgrade-${env.BUILD_NUMBER}" // Suite/method selectors passed as GOTEST_TESTCASE to the downstream executor. // Each phase is its own go test invocation (-run regex), so cross-phase state // lives in-cluster under deterministic names, never in process memory. -def PRE_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade"' -def POST_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade"' -def CHART_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade"' +// TEST_MODULE selects the entry (checkout stage fails fast on an unknown name). +// fleet ships inside Rancher with no standalone chart, so its map has no chart +// phase and the chart-upgrade stage is skipped for it. +def UPGRADE_TEST_MODULES = [ + longhorn: [ + pre: '-run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade"', + post: '-run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade"', + chart: '-run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade"' + ], + neuvector: [ + pre: '-run "TestNeuVectorUpgradeTestSuite/TestNeuVectorPreUpgrade"', + post: '-run "TestNeuVectorUpgradeTestSuite/TestNeuVectorPostUpgrade"', + chart: '-run "TestNeuVectorUpgradeTestSuite/TestNeuVectorChartUpgrade"' + ], + fleet: [ + pre: '-run "TestFleetUpgradeTestSuite/TestFleetPreUpgrade"', + post: '-run "TestFleetUpgradeTestSuite/TestFleetPostUpgrade"' + ], + observability: [ + pre: '-run "TestObservabilityUpgradeTestSuite/TestObservabilityPreUpgrade"', + post: '-run "TestObservabilityUpgradeTestSuite/TestObservabilityPostUpgrade"', + chart: '-run "TestObservabilityUpgradeTestSuite/TestObservabilityChartUpgrade"' + ] +] + +// Binding variable (no def) so every stage's script/when block sees it; set in +// the Checkout stage, read by the three phase stages below. +TEST_PHASES = null /** * Shared teardown helper: re-renders tfvars from params, tears down tofu workspace @@ -308,6 +334,11 @@ server_flags: | ) // ── Test parameters ──────────────────────────────────── + string( + name: 'TEST_MODULE', + defaultValue: params.TEST_MODULE ?: 'longhorn', + description: 'rancherupgrade module under test (longhorn | neuvector | fleet | observability). Selects the per-phase -run regexes; fleet has no chart-upgrade phase.' + ) text( name: 'CATTLE_TEST_CONFIG', description: 'Test configuration YAML (cattle-config). rancher.adminToken is injected at runtime (initial + rotated after the server upgrade); rancherUpgradeInput.targetVersion is substituted from the resolved upgrade target.', @@ -393,6 +424,15 @@ rancherUpgradePipeline: env.TESTS_DIR = dirs.testsDir env.INFRA_DIR = dirs.infraDir + // Resolve the selected module's phase selectors before any + // infra spend: an unknown TEST_MODULE fails here, and the + // log records exactly which regex each stage will run. + TEST_PHASES = UPGRADE_TEST_MODULES[params.TEST_MODULE] + if (!TEST_PHASES) { + error "Unknown TEST_MODULE '${params.TEST_MODULE}' — valid modules: ${UPGRADE_TEST_MODULES.keySet().sort().join(', ')}" + } + echo "TEST_MODULE '${params.TEST_MODULE}' phase selectors: pre=${TEST_PHASES.pre} post=${TEST_PHASES.post} chart=${TEST_PHASES.chart ?: '(none: module has no chart-upgrade phase)'}" + def hostnamePrefix = params.HOSTNAME_PREFIX ?: 'unknown' currentBuild.displayName = "#${env.BUILD_NUMBER} - ${hostnamePrefix}" currentBuild.description = "Rancher server upgrade validation for ${hostnamePrefix}" @@ -502,7 +542,7 @@ rancherUpgradePipeline: TO (upgrade): ${env.RANCHER_TO_VERSION} (${env.RANCHER_TO_IMAGE_TAG}) Chart flavor: ${params.RANCHER_CHART_REPO_FLAVOR} """.stripIndent() - currentBuild.description = "Upgrade ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (${params.RANCHER_CHART_REPO_FLAVOR}) | ${params.HOSTNAME_PREFIX}" + currentBuild.description = "Upgrade ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (${params.RANCHER_CHART_REPO_FLAVOR}) | ${params.HOSTNAME_PREFIX} | ${params.TEST_MODULE}" } } } @@ -758,9 +798,9 @@ rancher_use_bundled_system_charts: true } } - // ── Phase 1: pre-upgrade Longhorn install + workload seed ── - // Starts the shared Qase run once; all three phases report into it. - stage('Pre-Upgrade Longhorn Tests') { + // ── Phase 1: pre-upgrade module install + state seed ── + // Starts the shared Qase run once; every phase reports into it. + stage('Pre-Upgrade Tests') { steps { script { property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { @@ -802,7 +842,7 @@ rancher_use_bundled_system_charts: true // teardown is the cleanup. def configYaml = readFile(file: cattleConfigPath) - echo "Dispatching pre-upgrade phase: GOTEST_TESTCASE ${PRE_UPGRADE_TESTCASE}" + echo "Dispatching pre-upgrade phase: GOTEST_TESTCASE ${TEST_PHASES.pre}" build job: params.INDIVIDUAL_JOB, parameters: [ string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), text(name: 'CONFIG', value: configYaml), @@ -812,7 +852,7 @@ rancher_use_bundled_system_charts: true string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), - string(name: 'GOTEST_TESTCASE', value: PRE_UPGRADE_TESTCASE), + string(name: 'GOTEST_TESTCASE', value: TEST_PHASES.pre), string(name: 'TAGS', value: 'validation'), booleanParam(name: 'CAPTURE_IMAGES', value: false) ] @@ -881,7 +921,7 @@ rancher_use_bundled_system_charts: true } // ── Phase 3: post-upgrade assertions ── - stage('Post-Upgrade Longhorn Tests') { + stage('Post-Upgrade Tests') { steps { script { property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { @@ -904,7 +944,7 @@ rancher_use_bundled_system_charts: true echo "Dispatched config carries the rotated admin token (length \${#expected})" """ - echo "Dispatching post-upgrade phase: GOTEST_TESTCASE ${POST_UPGRADE_TESTCASE} (Qase run ${testRunID})" + echo "Dispatching post-upgrade phase: GOTEST_TESTCASE ${TEST_PHASES.post} (Qase run ${testRunID})" build job: params.INDIVIDUAL_JOB, parameters: [ string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), text(name: 'CONFIG', value: configYaml), @@ -914,19 +954,34 @@ rancher_use_bundled_system_charts: true string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), - string(name: 'GOTEST_TESTCASE', value: POST_UPGRADE_TESTCASE), + string(name: 'GOTEST_TESTCASE', value: TEST_PHASES.post), string(name: 'TAGS', value: 'validation'), booleanParam(name: 'CAPTURE_IMAGES', value: false) ] + + // Modules without a chart-upgrade phase (fleet) end + // their pipeline here: close the shared Qase run now. + // The post{always} fallback below still covers every + // failure path for every module. + if (!TEST_PHASES.chart) { + sh "./testrun --startRun=false" + env.QASE_RUN_COMPLETED = 'true' + echo "Shared Qase test run ${testRunID} completed (module has no chart-upgrade phase)" + } } } } } - // ── Phase 4: Longhorn chart upgrade through the upgraded catalog ── + // ── Phase 4: module chart upgrade through the upgraded catalog ── // Completes the shared Qase run after this final phase; do not rely on // the downstream job's own QASE_TEST_RUN_COMPLETE for the shared run. - stage('Longhorn Chart Upgrade Tests') { + // Skipped entirely for modules without a chart phase (fleet): their + // post-upgrade stage closes the shared Qase run instead. + stage('Chart-Upgrade Tests') { + when { + expression { TEST_PHASES?.chart } + } steps { script { property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { @@ -936,7 +991,7 @@ rancher_use_bundled_system_charts: true ).trim() def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") - echo "Dispatching chart-upgrade phase: GOTEST_TESTCASE ${CHART_UPGRADE_TESTCASE} (Qase run ${testRunID})" + echo "Dispatching chart-upgrade phase: GOTEST_TESTCASE ${TEST_PHASES.chart} (Qase run ${testRunID})" build job: params.INDIVIDUAL_JOB, parameters: [ string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), text(name: 'CONFIG', value: configYaml), @@ -946,7 +1001,7 @@ rancher_use_bundled_system_charts: true string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), - string(name: 'GOTEST_TESTCASE', value: CHART_UPGRADE_TESTCASE), + string(name: 'GOTEST_TESTCASE', value: TEST_PHASES.chart), string(name: 'TAGS', value: 'validation'), booleanParam(name: 'CAPTURE_IMAGES', value: false) ] From 4eb735899c021c910afab22cb77b059960f259e8 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 22 Oct 2026 12:30:27 -0500 Subject: [PATCH 17/21] Thread the node SSH key through the test config for the fleet module The fleet suite serves its GitRepo over SSH from a downstream node, but downstream executors mount no .ssh, so the suite cannot reach a key by itself. For TEST_MODULE=fleet the config-creation stage injects the orchestrator's node PEM and the tfvars ssh user into the sshNode section of the cattle-config; other modules are unaffected. --- validation/Jenkinsfile.rancher-upgrade.e2e | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 3c406390c8..0256e006e1 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -741,6 +741,36 @@ rancher_use_bundled_system_charts: true writeFile file: cattleConfigPath, text: resolvedConfig.trim() + '\n' } + // The fleet module's pre-upgrade phase serves its GitRepo + // over SSH from a downstream node. Downstream executors + // mount no .ssh, so the node key is threaded through the + // test config for fleet alone; the PEM sits in the + // orchestrator workspace where writeSshKey put it for + // ansible. + if (params.TEST_MODULE == 'fleet') { + property.useWithProperties(['AWS_SSH_PEM_KEY_NAME']) { + def pemPath = "${workspace}/.ssh/${AWS_SSH_PEM_KEY_NAME}" + if (!fileExists(pemPath)) { + error "fleet module needs the node key at ${pemPath} (AWS_SSH_PEM_KEY_NAME), but it does not exist" + } + def sshUser = sh( + script: "awk -F': *' '/^aws_ssh_user/{print \$2}' ${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes/terraform.tfvars | head -1 | tr -d '\"'", + returnStdout: true + ).trim() + if (!sshUser) { + sshUser = 'ubuntu' + } + sh """ + set +x + export nodekey=\$(cat ${pemPath}) + export sshuser='${sshUser}' + ./yq e '.sshNode.sshKey = strenv(nodekey)' -i ${cattleConfigPath} + ./yq e '.sshNode.sshUser = strenv(sshuser)' -i ${cattleConfigPath} + """ + echo "fleet module: node ssh key and user threaded into the test config" + } + } + // Generate admin token via Ansible playbook // TODO: replace with 'make token' once that target is added upstream def containerConfigPath = "/workspace/${cattleConfigPath}" From 7d4b2842579c4b2685530da5ef7b4dbf21b95238 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 22 Oct 2026 21:58:17 -0500 Subject: [PATCH 18/21] Thread the StackState connection settings for the observability module The observability suite requires the stackstateConfigs operator inputs (license, url, cluster API key, admin password); the pipeline injects them for TEST_MODULE=observability: the license from the STACKSTATE_LICENSE folder credential (failing fast before infra spend when absent), a STACKSTATE_URL parameter defaulting to the in-cluster router service the co-located agent can always reach, and a self-consistent cluster API key the suite installs the server with. --- validation/Jenkinsfile.rancher-upgrade.e2e | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 0256e006e1..5fee83d686 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -339,6 +339,11 @@ server_flags: | defaultValue: params.TEST_MODULE ?: 'longhorn', description: 'rancherupgrade module under test (longhorn | neuvector | fleet | observability). Selects the per-phase -run regexes; fleet has no chart-upgrade phase.' ) + string( + name: 'STACKSTATE_URL', + defaultValue: params.STACKSTATE_URL ?: 'http://stackstate-router.suse-observability.svc.cluster.local:8080', + description: 'observability module only: the URL the StackState agent reports to. The in-cluster router service default works when the agent is co-located with the server (the pipeline pins the server to local); override with a resolvable StackState address otherwise.' + ) text( name: 'CATTLE_TEST_CONFIG', description: 'Test configuration YAML (cattle-config). rancher.adminToken is injected at runtime (initial + rotated after the server upgrade); rancherUpgradeInput.targetVersion is substituted from the resolved upgrade target.', @@ -771,6 +776,31 @@ rancher_use_bundled_system_charts: true } } + if (params.TEST_MODULE == 'observability') { + // SUSE Observability needs its license plus the + // connection settings the server and the agent share. + // The license is a folder credential (fail fast here, + // before any infra spend); the URL defaults to the + // in-cluster router service, which the co-located + // agent can always reach; the cluster API key is + // self-consistent because this suite installs the + // server with the same value. + property.useWithProperties(['STACKSTATE_LICENSE']) { + if (!env.STACKSTATE_LICENSE) { + error "observability module needs the STACKSTATE_LICENSE folder credential (SUSE Observability license key); add it to the rancher_qa folder and rerun" + } + sh """ + set +x + export sslicense=\$STACKSTATE_LICENSE + ./yq e '.stackstateConfigs.url = "${params.STACKSTATE_URL}"' -i ${cattleConfigPath} + ./yq e '.stackstateConfigs.license = strenv(sslicense)' -i ${cattleConfigPath} + ./yq e '.stackstateConfigs.clusterApiKey = "rancher-upgrade-observability-cluster-key"' -i ${cattleConfigPath} + ./yq e '.stackstateConfigs.adminPassword = "rancher-upgrade-observability-admin"' -i ${cattleConfigPath} + """ + echo "observability module: stackstate connection settings threaded into the test config" + } + } + // Generate admin token via Ansible playbook // TODO: replace with 'make token' once that target is added upstream def containerConfigPath = "/workspace/${cattleConfigPath}" From 579878d9b7a4712a06f99d57522cecb476e00bf4 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Fri, 23 Oct 2026 00:45:17 -0500 Subject: [PATCH 19/21] Guard the StackState license credential before deploy Build 24 showed the license check firing only after the pre-upgrade environment was already deployed, burning the deploy on a missing credential. The version-resolution stage now verifies STACKSTATE_LICENSE exists and is non-empty before any infra spend; the post-deploy config injection reuses it. --- validation/Jenkinsfile.rancher-upgrade.e2e | 24 ++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 5fee83d686..679ce23dac 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -548,6 +548,25 @@ rancherUpgradePipeline: Chart flavor: ${params.RANCHER_CHART_REPO_FLAVOR} """.stripIndent() currentBuild.description = "Upgrade ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (${params.RANCHER_CHART_REPO_FLAVOR}) | ${params.HOSTNAME_PREFIX} | ${params.TEST_MODULE}" + + if (params.TEST_MODULE == 'observability') { + // Pre-deploy guard: fail before any infra spend when + // the SUSE Observability license credential is absent + // or empty; the post-deploy config injection reuses it. + def licenseProblem = '' + try { + withCredentials([string(credentialsId: 'STACKSTATE_LICENSE', variable: 'STACKSTATE_LICENSE_CHECK')]) { + if (!env.STACKSTATE_LICENSE_CHECK?.trim()) { + licenseProblem = 'STACKSTATE_LICENSE exists but is empty; set the SUSE Observability license key and rerun' + } + } + } catch (Exception e) { + licenseProblem = "observability module needs the STACKSTATE_LICENSE folder credential (SUSE Observability license key); add it to the rancher_qa folder and rerun (checked pre-deploy to avoid infra spend)" + } + if (licenseProblem) { + error licenseProblem + } + } } } } @@ -779,8 +798,9 @@ rancher_use_bundled_system_charts: true if (params.TEST_MODULE == 'observability') { // SUSE Observability needs its license plus the // connection settings the server and the agent share. - // The license is a folder credential (fail fast here, - // before any infra spend); the URL defaults to the + // The license is the STACKSTATE_LICENSE folder + // credential, checked pre-deploy by the guard in the + // version-resolution stage; the URL defaults to the // in-cluster router service, which the co-located // agent can always reach; the cluster API key is // self-consistent because this suite installs the From 223ce7aebbdc9f43877bbf8579bf500907e2ef37 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 27 Oct 2026 07:08:26 -0500 Subject: [PATCH 20/21] Run every rancherupgrade module in one pipeline (TEST_MODULE=all) Add an all entry to UPGRADE_TEST_MODULES whose broad per-phase regex picks up every module suite in the package (and future ones); thread the fleet ssh key and the observability license when TEST_MODULE=all, and pre-deploy-guard the license for it too. Fleet has no chart subtest, so the chart phase runs its Setup/TearDown with zero matched methods. --- validation/Jenkinsfile.rancher-upgrade.e2e | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 679ce23dac..66bbc0f538 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -76,6 +76,15 @@ def UPGRADE_TEST_MODULES = [ pre: '-run "TestObservabilityUpgradeTestSuite/TestObservabilityPreUpgrade"', post: '-run "TestObservabilityUpgradeTestSuite/TestObservabilityPostUpgrade"', chart: '-run "TestObservabilityUpgradeTestSuite/TestObservabilityChartUpgrade"' + ], + all: [ + // Every module's suite in one pipeline run; the broad regex also + // picks up modules added to the package later. Fleet has no chart + // subtest, so the chart phase runs its Setup/TearDown with zero + // matched methods (harmless: its config keys are still threaded). + pre: '-run "Test.*UpgradeTestSuite/Test.*PreUpgrade"', + post: '-run "Test.*UpgradeTestSuite/Test.*PostUpgrade"', + chart: '-run "Test.*UpgradeTestSuite/Test.*ChartUpgrade"' ] ] @@ -337,7 +346,7 @@ server_flags: | string( name: 'TEST_MODULE', defaultValue: params.TEST_MODULE ?: 'longhorn', - description: 'rancherupgrade module under test (longhorn | neuvector | fleet | observability). Selects the per-phase -run regexes; fleet has no chart-upgrade phase.' + description: 'rancherupgrade module under test (longhorn | neuvector | fleet | observability | all). Selects the per-phase -run regexes; fleet has no chart-upgrade phase; all runs every module suite in one pipeline.' ) string( name: 'STACKSTATE_URL', @@ -549,7 +558,7 @@ rancherUpgradePipeline: """.stripIndent() currentBuild.description = "Upgrade ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (${params.RANCHER_CHART_REPO_FLAVOR}) | ${params.HOSTNAME_PREFIX} | ${params.TEST_MODULE}" - if (params.TEST_MODULE == 'observability') { + if (params.TEST_MODULE in ['observability', 'all']) { // Pre-deploy guard: fail before any infra spend when // the SUSE Observability license credential is absent // or empty; the post-deploy config injection reuses it. @@ -771,7 +780,7 @@ rancher_use_bundled_system_charts: true // test config for fleet alone; the PEM sits in the // orchestrator workspace where writeSshKey put it for // ansible. - if (params.TEST_MODULE == 'fleet') { + if (params.TEST_MODULE in ['fleet', 'all']) { property.useWithProperties(['AWS_SSH_PEM_KEY_NAME']) { def pemPath = "${workspace}/.ssh/${AWS_SSH_PEM_KEY_NAME}" if (!fileExists(pemPath)) { @@ -795,7 +804,7 @@ rancher_use_bundled_system_charts: true } } - if (params.TEST_MODULE == 'observability') { + if (params.TEST_MODULE in ['observability', 'all']) { // SUSE Observability needs its license plus the // connection settings the server and the agent share. // The license is the STACKSTATE_LICENSE folder From c67f600ff1faa3cfb4bdf5ea391de2e2cbe7e9f8 Mon Sep 17 00:00:00 2001 From: dnewman Date: Tue, 27 Oct 2026 13:34:21 -0500 Subject: [PATCH 21/21] Sequence the all-mode chart phase observability-first Take the Jenkinsfile from the integration branch verbatim: in all-mode dispatch the observability chart upgrade before longhorn's (Longhorn's engine live-upgrades stall a concurrent observability rollout's volume reattach; build 52). --- validation/Jenkinsfile.rancher-upgrade.e2e | 45 +++++++++++++++------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 66bbc0f538..1c72ba46c7 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -1080,20 +1080,37 @@ rancher_use_bundled_system_charts: true ).trim() def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") - echo "Dispatching chart-upgrade phase: GOTEST_TESTCASE ${TEST_PHASES.chart} (Qase run ${testRunID})" - build job: params.INDIVIDUAL_JOB, parameters: [ - string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), - text(name: 'CONFIG', value: configYaml), - string(name: 'REPO', value: params.TESTS_REPO_URL), - string(name: 'BRANCH', value: params.TESTS_BRANCH), - string(name: 'QASE_TEST_RUN_ID', value: testRunID), - string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), - string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), - string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), - string(name: 'GOTEST_TESTCASE', value: TEST_PHASES.chart), - string(name: 'TAGS', value: 'validation'), - booleanParam(name: 'CAPTURE_IMAGES', value: false) - ] + // All-mode sequencing: the volume-heavy observability + // chart upgrade runs BEFORE longhorn's. Longhorn's + // upgrade live-upgrades every volume engine in the + // cluster, and an observability rollout starting while + // those engine upgrades are in flight can stall its + // statefulsets on volume reattach for the full helm + // deadline (build 52: zookeeper stuck in + // ContainerCreating for 45 minutes, kafka crashlooping + // on the unresolved headless service, everything + // cascading from there). + def chartSelectors = params.TEST_MODULE == 'all' ? [ + UPGRADE_TEST_MODULES.observability.chart, + '-run "Test(Fleet|Longhorn|NeuVector)UpgradeTestSuite/Test(Fleet|Longhorn|NeuVector)ChartUpgrade"' + ] : [TEST_PHASES.chart] + + chartSelectors.each { selector -> + echo "Dispatching chart-upgrade phase: GOTEST_TESTCASE ${selector} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: selector), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } // Complete the shared qase test run sh "./testrun --startRun=false"