From 36fcdf06459fce0338023f5f1eed5a0640ddf5f8 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 7 Oct 2026 21:41:03 -0500 Subject: [PATCH 01/19] Add rancherupgrade package: Longhorn pre/post/chart-upgrade validation Implements the Longhorn vertical of the Rancher server upgrade validation (rancher/tests#967, parent PRD #946). New validation/rancherupgrade package with a LonghornUpgradeTestSuite whose phase methods run as separate go test invocations selected via -run, one per pipeline stage: - TestLonghornPreUpgrade: create project, install the Longhorn chart at the cluster-scoped catalog latest, create a checksum workload (PVC + nginx deployment with deterministic names) on a Longhorn volume. - TestLonghornPostUpgrade: gate on the management server-version setting reaching rancherUpgradeInput.targetVersion, assert the Longhorn chart survived the server upgrade, verify the checksum workload data. - TestLonghornChartUpgrade: gate on server version, resolve latest from the cluster-scoped catalog, upgrade longhorn + longhorn-crd, assert the installed chart version, verify the checksum workload again. Supporting actions: - actions/clusters: WaitRancherVersion polls the server-version setting until it semver-equals the target, retrying transient read errors. - actions/storage: CreateChecksumWorkload/VerifyChecksumWorkload build and verify a deterministically named PVC/deployment pair whose sha256-verified data file must survive across processes; no session cleanup is registered by design. - actions/charts: UpgradeLonghornChart upgrades longhorn and longhorn-crd with the deletingConfirmationFlag value preserved and OpenAPI validation disabled, waits for Deployed, then gates on deployments and daemonsets. TearDownSuite deliberately skips session.Cleanup and the suite is admin-only: upgrade-subject resources must survive into later phases and the pipeline environment teardown is the cleanup. Both deviations are documented in the package README, along with the token-rotation and catalog-follower-restart notes for the pipeline. Includes schemas/pit_schemas.yaml for the three phase methods. Verified live against a 2.14.3 HA env with a node-driver downstream RKE2 cluster: pre-upgrade PASS (Longhorn 109.3.1+up1.11.2, checksum seeded), server upgraded to 2.15.2 via qa-infra-automation, post-upgrade PASS, chart upgrade PASS (109.3.1+up1.11.2 -> 110.1.0+up1.12.1, checksum intact). --- actions/charts/longhorn.go | 118 +++++++++ actions/clusters/local.go | 36 +++ actions/storage/checksum.go | 234 ++++++++++++++++++ validation/rancherupgrade/README.md | 82 ++++++ validation/rancherupgrade/longhorn_test.go | 145 +++++++++++ validation/rancherupgrade/rancherupgrade.go | 18 ++ .../rancherupgrade/schemas/pit_schemas.yaml | 61 +++++ 7 files changed, 694 insertions(+) create mode 100644 actions/storage/checksum.go create mode 100644 validation/rancherupgrade/README.md create mode 100644 validation/rancherupgrade/longhorn_test.go create mode 100644 validation/rancherupgrade/rancherupgrade.go create mode 100644 validation/rancherupgrade/schemas/pit_schemas.yaml diff --git a/actions/charts/longhorn.go b/actions/charts/longhorn.go index 89c8640cb3..c1191e3859 100644 --- a/actions/charts/longhorn.go +++ b/actions/charts/longhorn.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" "github.com/rancher/shepherd/clients/rancher" "github.com/rancher/shepherd/clients/rancher/catalog" shepherdCharts "github.com/rancher/shepherd/extensions/charts" @@ -100,6 +101,123 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma return err } +// UpgradeLonghornChart upgrades the longhorn and longhorn-crd charts to +// installOptions.Version on installOptions.Cluster and waits for Deployed state. +// No session cleanup is registered: the uninstall cleanup belongs to the install +// call of the same session, and upgrade-test sessions must not delete the chart. +func UpgradeLonghornChart(client *rancher.Client, installOptions *InstallOptions) error { + serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) + if err != nil { + return err + } + + registrySetting, err := client.Management.Setting.ByID(defaultRegistrySettingID) + if err != nil { + return err + } + + longhornChartUpgradeActionPayload := &PayloadOpts{ + InstallOptions: *installOptions, + Name: LonghornChartName, + Namespace: LonghornNamespace, + Host: serverSetting.Value, + DefaultRegistry: registrySetting.Value, + } + + chartUpgradeAction := newLonghornChartUpgradeAction(longhornChartUpgradeActionPayload) + + catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + bodyBytes, err := json.Marshal(chartUpgradeAction) + if err != nil { + return err + } + err = ChartActionWithRetry(context.TODO(), client, verbUpgrade, longhornChartUpgradeActionPayload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbUpgrade, bodyBytes)) + if err != nil { + return err + } + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + if err != nil { + return err + } + + adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + watchAppInterface, err := adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + LonghornChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusPendingUpgrade) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + watchAppInterface, err = adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + LonghornChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusDeployed) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{}) + if err != nil { + return err + } + + return shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{}) +} + +// newLonghornChartUpgradeAction is a private helper function that returns the chart upgrade action for Longhorn. +func newLonghornChartUpgradeAction(p *PayloadOpts) *types.ChartUpgradeAction { + // Pass the deletingConfirmationFlag value on the main chart so the upgrade does + // not reset it; the CRD chart takes no Longhorn settings (same split as install). + chartUpgrade := NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, enableDeletionSetting) + chartUpgradeCRD := NewChartUpgrade(p.Name+"-crd", p.Name+"-crd", p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, nil) + chartUpgrades := []types.ChartUpgrade{*chartUpgradeCRD, *chartUpgrade} + + chartUpgradeAction := NewChartUpgradeAction(p.Namespace, chartUpgrades) + // Disable OpenAPI validation to avoid schema-conflict failures on the large + // Longhorn CRDs, which frequently change shape between versions. + chartUpgradeAction.DisableOpenAPIValidation = true + + return chartUpgradeAction +} + // UninstallLonghornChart removes Longhorn from the cluster related to the received catalog client object. func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID string, rancherHostname string) error { catalogClient, err := client.GetClusterCatalogClient(clusterID) diff --git a/actions/clusters/local.go b/actions/clusters/local.go index 206e523b81..752df582ee 100644 --- a/actions/clusters/local.go +++ b/actions/clusters/local.go @@ -1,13 +1,18 @@ package clusters import ( + "context" "errors" "fmt" "regexp" "strings" + "time" "github.com/Masterminds/semver/v3" "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -37,6 +42,37 @@ func IsRancherVersionAbove(client *rancher.Client, minVersion string) (bool, err return serverVersion.GreaterThan(minimumVersion), nil } +// WaitRancherVersion polls the management server-version setting until it equals +// targetVersion (semver-compared) or timeout elapses. Gates post-upgrade assertions +// on Rancher server convergence; transient read errors during the upgrade rollout +// are retried, not returned. +func WaitRancherVersion(client *rancher.Client, targetVersion string, timeout time.Duration) error { + target, err := parseRancherVersion(targetVersion) + if err != nil { + return err + } + + return wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, timeout, true, func(ctx context.Context) (done bool, err error) { + serverVersionRawValue, err := client.Management.Setting.ByID(serverVersionSetting) + if err != nil { + // The server may be mid-restart during the upgrade rollout; retry the read. + return false, nil + } + + serverVersion, err := parseRancherVersion(serverVersionRawValue.Value) + if err != nil { + return false, nil + } + + if serverVersion.Equal(target) { + return true, nil + } + + logrus.Debugf("Rancher server version is %q, waiting for %q", serverVersionRawValue.Value, targetVersion) + return false, nil + }) +} + func parseRancherVersion(versionInput string) (*semver.Version, error) { trimmedVersion := strings.TrimSpace(strings.TrimPrefix(versionInput, "v")) if trimmedVersion == "" { diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go new file mode 100644 index 0000000000..82ab4d305c --- /dev/null +++ b/actions/storage/checksum.go @@ -0,0 +1,234 @@ +package storage + +import ( + "context" + "crypto/sha256" + "fmt" + "strings" + + "github.com/rancher/shepherd/clients/rancher" + steveV1 "github.com/rancher/shepherd/clients/rancher/v1" + "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/extensions/defaults/namespaces" + "github.com/rancher/shepherd/extensions/defaults/stevetypes" + "github.com/rancher/shepherd/extensions/kubeapi/cluster" + "github.com/rancher/shepherd/extensions/kubeconfig" + wloads "github.com/rancher/shepherd/extensions/workloads" + "github.com/rancher/tests/actions/kubeapi/volumes/persistentvolumeclaims" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" +) + +// ChecksumDataFile is the filename written under MountPath by CreateChecksumWorkload. +const ChecksumDataFile = "checksum-data" + +// CreateChecksumWorkload creates a 1GiB PVC named on storageClassName in the +// default namespace, mounts it into a single-replica nginx deployment named at +// MountPath, waits for rollout, then writes content to MountPath/ChecksumDataFile via +// kubectl exec. Deterministic names let a later test phase (running as a separate +// process) relocate the resources. content must be shell-safe (alphanumeric/hyphen); +// it is interpolated into an sh -c command. No session cleanup is registered: callers +// are upgrade tests whose resources must outlive the process. +func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageClassName string, name string, content string) error { + storageClass, err := GetStorageClass(client, clusterID, storageClassName) + if err != nil { + return err + } + + wrangler, err := cluster.GetClusterWranglerContext(client, clusterID) + if err != nil { + return err + } + + accessModes := []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + } + + // Deliberate deviation from CreatePVC: no PV-delete/Longhorn-volume cleanup funcs + // are registered, because these resources must survive into later upgrade phases. + _, err = persistentvolumeclaims.CreatePersistentVolumeClaim( + client, + clusterID, + name, + "test-pvc-volume", + namespaces.Default, + 1, + accessModes, + nil, + &storageClass, + ) + if err != nil { + return err + } + + pvc := &corev1.PersistentVolumeClaim{} + err = wait.PollUntilContextTimeout(context.Background(), pollInterval, defaults.OneMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + pvc, err = wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + return pvc.Status.Phase == persistentvolumeclaims.PersistentVolumeBoundStatus, nil + }) + if err != nil { + return err + } + logrus.Debugf("PVC %s is Bound to volume %s", pvc.Name, pvc.Spec.VolumeName) + + volMount := corev1.VolumeMount{ + MountPath: MountPath, + Name: name, + } + + podVol := corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: name, + }, + }, + } + + steveclient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return err + } + + containerTemplate := wloads.NewContainer(nginxName, nginxName, corev1.PullAlways, []corev1.VolumeMount{volMount}, []corev1.EnvFromSource{}, nil, nil, nil) + podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: name}, nil) + deployment := wloads.NewDeploymentTemplate(name, namespaces.Default, podTemplate, true, nil) + + _, err = steveclient.SteveType(stevetypes.Deployment).Create(deployment) + if err != nil { + return err + } + + err = charts.WatchAndWaitDeployments(client, clusterID, namespaces.Default, metav1.ListOptions{ + FieldSelector: "metadata.name=" + name, + }) + if err != nil { + return err + } + + pod, err := getChecksumPod(client, clusterID, name) + if err != nil { + return err + } + + restConfig, err := getDownstreamRestConfig(client, clusterID) + if err != nil { + return err + } + + writeCommand := []string{"sh", "-c", "printf '%s' '" + content + "' > " + MountPath + "/" + ChecksumDataFile} + logrus.Debugf("Writing %s/%s on pod %s", MountPath, ChecksumDataFile, pod.Name) + _, err = kubeconfig.KubectlExec(restConfig, pod.Name, namespaces.Default, writeCommand) + + return err +} + +// VerifyChecksumWorkload relocates the PVC and pod created by CreateChecksumWorkload, +// asserts the PVC is still Bound, and asserts the sha256 of MountPath/ChecksumDataFile +// inside the running pod equals sha256(content). +func VerifyChecksumWorkload(client *rancher.Client, clusterID string, name string, content string) error { + wrangler, err := cluster.GetClusterWranglerContext(client, clusterID) + if err != nil { + return err + } + + pvc, err := wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) + if err != nil { + return err + } + + if pvc.Status.Phase != persistentvolumeclaims.PersistentVolumeBoundStatus { + return fmt.Errorf("PVC %s is %s, expected %s", name, pvc.Status.Phase, persistentvolumeclaims.PersistentVolumeBoundStatus) + } + + pod, err := getChecksumPod(client, clusterID, name) + if err != nil { + return err + } + + restConfig, err := getDownstreamRestConfig(client, clusterID) + if err != nil { + return err + } + + checksumCommand := []string{"sha256sum", MountPath + "/" + ChecksumDataFile} + output, err := kubeconfig.KubectlExec(restConfig, pod.Name, namespaces.Default, checksumCommand) + if err != nil { + return err + } + + // The exec runs with a TTY, so the output carries \r line endings; strip them + // before extracting the hash (first whitespace-separated field). + outputFields := strings.Fields(strings.ReplaceAll(output.String(), "\r", "")) + if len(outputFields) == 0 { + return fmt.Errorf("empty sha256sum output on pod %s", pod.Name) + } + + actualChecksum := outputFields[0] + expectedChecksum := fmt.Sprintf("%x", sha256.Sum256([]byte(content))) + if actualChecksum != expectedChecksum { + return fmt.Errorf("checksum mismatch on pod %s: expected %s, got %s", pod.Name, expectedChecksum, actualChecksum) + } + + return nil +} + +// getChecksumPod returns the pod in the default namespace labeled with +// DeploymentIdentifierLabel == name, waiting until it exists and is Running. +func getChecksumPod(client *rancher.Client, clusterID string, name string) (*steveV1.SteveAPIObject, error) { + var checksumPod *steveV1.SteveAPIObject + err := wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, defaults.TwoMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + steveclient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return false, err + } + + podsResp, err := steveclient.SteveType(stevetypes.Pod).NamespacedSteveClient(namespaces.Default).List(nil) + if err != nil { + return false, err + } + + for i := range podsResp.Data { + pod := &podsResp.Data[i] + if pod.Labels[DeploymentIdentifierLabel] != name { + continue + } + + podStatus := &corev1.PodStatus{} + err = steveV1.ConvertToK8sType(pod.Status, podStatus) + if err != nil { + return false, err + } + + if podStatus.Phase != corev1.PodRunning { + logrus.Debugf("Pod %s is %s, waiting for Running", pod.Name, podStatus.Phase) + return false, nil + } + + checksumPod = pod + return true, nil + } + + return false, nil + }) + + return checksumPod, err +} + +// getDownstreamRestConfig builds a rest config for kubectl exec against the cluster. +func getDownstreamRestConfig(client *rancher.Client, clusterID string) (*rest.Config, error) { + kubeConfig, err := kubeconfig.GetKubeconfig(client, clusterID) + if err != nil { + return nil, err + } + + return (*kubeConfig).ClientConfig() +} diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md new file mode 100644 index 0000000000..7436559c6b --- /dev/null +++ b/validation/rancherupgrade/README.md @@ -0,0 +1,82 @@ +# Rancher Upgrade + +The `rancherupgrade` package validates upgrading the Rancher server (2.14.x -> latest) +with interoperability products installed. The first vertical is Longhorn: a pre-upgrade +phase installs the Longhorn chart and seeds a checksum workload on a Longhorn volume, +the Rancher server is then upgraded out-of-band (qa-infra-automation `make rancher-upgrade`), +and post-upgrade phases verify the server version, the surviving Longhorn installation, +the workload data, and a Longhorn chart upgrade to the version offered by the upgraded +Rancher's catalog. + +Each phase is a suite test method selected via the `-run` regex and runs as its own +`go test` invocation in its own pipeline stage. All cross-phase state lives in-cluster +under deterministic names (PVC/deployment `rancher-upgrade-longhorn`, data file +`/auto-mnt/checksum-data`), never in process memory. + +## Deliberate deviations + +Do not "fix" the following; both are required by the upgrade workflow (rancher/tests#967): + +- `TearDownSuite` does not call `session.Cleanup()` and the helpers register no cleanup + funcs: pre-phase resources (Longhorn chart, PVC, workload) must survive into the + later phases, which run as separate processes. The pipeline environment teardown is + the cleanup. +- The suite is admin-only: upgrade phases assert server-level state and the pipeline + runs them with the admin token. + +## Config + +```yaml +rancher: + host: + adminToken: + insecure: true + clusterName: "" # pre-provisioned by qa-infra-automation + +rancherUpgradeInput: + targetVersion: "" +``` + +`rancher.clusterName` selects the shared downstream cluster. `rancherUpgradeInput.targetVersion` +is required by the post-upgrade and chart-upgrade phases; the pipeline writes it from +the qa-infra-automation resolver output, or set it by hand for local runs. + +## Running + +Pre-upgrade (installs Longhorn at the 2.14-era latest and seeds the checksum workload; +run against the not-yet-upgraded Rancher): + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade" +``` + +Post-upgrade (after `make rancher-upgrade RANCHER_VERSION_TO_UPGRADE= RANCHER_CHART_REPO_FLAVOR=community|prime` +in qa-infra-automation, and with `rancherUpgradeInput.targetVersion` set): + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade" +``` + +Chart upgrade: + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade" +``` + +## Notes + +- The shared downstream RKE2 cluster is provisioned by qa-infra-automation and must be + sized for Longhorn plus the later NeuVector/Observability suites; this is an infra + concern and is not configurable here. +- The Rancher server upgrade itself is `make rancher-upgrade` in + [qa-infra-automation](https://github.com/rancher/qa-infra-automation); this package + only runs the pre- and post-upgrade assertions around it. +- `make rancher-upgrade` rotates the admin token and writes the new one to + `ansible/rancher/-ha/generated.tfvars`; the pipeline must copy it into the test + config's `rancher.adminToken` before the post-upgrade phases (the pre-upgrade token + is revoked by the upgrade). +- After a server upgrade the downstream cluster-scoped catalog follower can keep + serving the previous Rancher line's index (e.g. release-v2.14 after upgrading to + 2.15) until the downstream `cattle-cluster-agent` re-syncs. If the chart-upgrade + phase fails with "failed to find chart longhorn from the chart repo", restart the + downstream cluster-agent and retry; observed re-sync time ~4 minutes. diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go new file mode 100644 index 0000000000..379fb38b9e --- /dev/null +++ b/validation/rancherupgrade/longhorn_test.go @@ -0,0 +1,145 @@ +//go:build validation && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 + +package rancherupgrade + +import ( + "testing" + + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + "github.com/rancher/tests/actions/charts" + actionsClusters "github.com/rancher/tests/actions/clusters" + "github.com/rancher/tests/actions/projects" + "github.com/rancher/tests/actions/storage" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type LonghornUpgradeTestSuite struct { + suite.Suite + session *session.Session + client *rancher.Client + cluster *clusters.ClusterMeta +} + +const ( + // checksumWorkloadName is the deterministic name shared by the PVC and the nginx + // deployment created in the pre-upgrade phase and relocated by later phases. + checksumWorkloadName = "rancher-upgrade-longhorn" + + // checksumContent is the shell-safe data written to the checksum file on the + // Longhorn-backed volume before the Rancher upgrade and verified after it. + checksumContent = "rancher-upgrade-longhorn-checksum-v1" +) + +// loadUpgradeInput returns the rancherUpgradeInput section of the test config. +func loadUpgradeInput() *UpgradeInput { + upgradeInput := new(UpgradeInput) + config.LoadConfig(UpgradeInputConfigKey, upgradeInput) + + return upgradeInput +} + +func (s *LonghornUpgradeTestSuite) SetupSuite() { + testSession := session.NewSession() + s.session = testSession + + client, err := rancher.NewClient("", testSession) + require.NoError(s.T(), err) + s.client = client + + s.cluster, err = clusters.NewClusterMeta(client, client.RancherConfig.ClusterName) + require.NoError(s.T(), err) +} + +// TearDownSuite deliberately does not call session.Cleanup(): resources created in +// the pre-upgrade phase (Longhorn chart, PVC, workload) must survive into the +// post-upgrade and chart-upgrade phases, which run as separate go test processes. +// The pipeline environment teardown is the cleanup for these resources. +func (s *LonghornUpgradeTestSuite) TearDownSuite() { +} + +func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { + project, _, err := projects.CreateProjectAndNamespace(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + // Resolve the version from the cluster-scoped catalog and install that: the + // management view can offer versions the cluster-scoped shard does not serve + // yet (rancher-2.15-catalog-sharding-issue.md), and the chart action validates + // against the cluster-scoped index. The pre-phase still yields the 2.14-era + // version by construction, since it runs against the pre-upgrade Rancher. + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + version, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + err = charts.InstallLonghornChart(s.client, charts.PayloadOpts{ + Namespace: charts.LonghornNamespace, + Host: s.client.RancherConfig.Host, + InstallOptions: charts.InstallOptions{ + Cluster: s.cluster, + Version: version, + ProjectID: project.ID, + }, + }, nil) + require.NoError(s.T(), err) + + err = storage.CreateChecksumWorkload(s.client, s.cluster.ID, charts.LonghornStorageClass, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) +} + +func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { + cfg := loadUpgradeInput() + require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.True(s.T(), chart.IsAlreadyInstalled, "longhorn chart must still be installed after the Rancher server upgrade") + + err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) +} + +func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { + cfg := loadUpgradeInput() + require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + + // Resolve the latest version from the cluster-scoped catalog: the management + // view can report versions absent from the cluster-scoped index (see + // rancher-2.15-catalog-sharding-issue.md at the repo root). + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + latest, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + installOptions := &charts.InstallOptions{ + Cluster: s.cluster, + Version: latest, + } + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return charts.UpgradeLonghornChart(s.client, installOptions) + }) + require.NoError(s.T(), err) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.Equal(s.T(), latest, chart.ChartDetails.Spec.Chart.Metadata.Version) + + err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) +} + +func TestLonghornUpgradeTestSuite(t *testing.T) { + suite.Run(t, new(LonghornUpgradeTestSuite)) +} diff --git a/validation/rancherupgrade/rancherupgrade.go b/validation/rancherupgrade/rancherupgrade.go new file mode 100644 index 0000000000..a08cf104eb --- /dev/null +++ b/validation/rancherupgrade/rancherupgrade.go @@ -0,0 +1,18 @@ +// Package rancherupgrade validates the Rancher server upgrade path (2.14.x -> latest) +// with interoperability products installed. Each phase (pre-upgrade, post-upgrade, +// chart-upgrade) is a suite test method selected via the -run regex and runs as its +// own go test invocation, so all cross-phase state lives in-cluster under the +// deterministic names defined here. +package rancherupgrade + +const ( + // UpgradeInputConfigKey is the cattle-config key holding upgrade-phase inputs. + UpgradeInputConfigKey = "rancherUpgradeInput" +) + +// UpgradeInput holds per-run upgrade parameters. TargetVersion is the Rancher +// version the server is upgraded to; written into the config by the pipeline +// (qa-infra-automation resolver output) or by hand for local runs. +type UpgradeInput struct { + TargetVersion string `json:"targetVersion" yaml:"targetVersion"` +} diff --git a/validation/rancherupgrade/schemas/pit_schemas.yaml b/validation/rancherupgrade/schemas/pit_schemas.yaml new file mode 100644 index 0000000000..da764c080d --- /dev/null +++ b/validation/rancherupgrade/schemas/pit_schemas.yaml @@ -0,0 +1,61 @@ +- projects: [RANCHERINT] + suite: Rancher Upgrade/Longhorn + cases: + - title: "Longhorn pre-upgrade setup" + description: "Install Longhorn and seed a checksum workload on a Longhorn volume before the Rancher server upgrade" + automation: 2 + steps: + - action: "Create a project on the downstream cluster" + data: "" + expectedresult: "The project is created successfully" + position: 1 + - action: "Install the Longhorn chart at the latest version offered by the pre-upgrade Rancher" + data: "Chart: longhorn from the rancher-charts repo" + expectedresult: "Longhorn installs and its deployments and daemonsets become Ready" + position: 2 + - action: "Create the checksum workload on a Longhorn-backed PVC" + data: "1GiB PVC and nginx deployment named rancher-upgrade-longhorn with data file /auto-mnt/checksum-data" + expectedresult: "PVC is Bound, deployment rolls out, and the checksum data is written to the Longhorn volume" + position: 3 + custom_field: + "15": "TestLonghornPreUpgrade" + - title: "Longhorn post-upgrade validation" + description: "Verify Longhorn and its volume data survive the Rancher server upgrade" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Check the Longhorn chart installation state" + data: "Chart: longhorn in longhorn-system" + expectedresult: "The chart is still installed" + position: 2 + - action: "Verify the checksum workload on the Longhorn volume" + data: "PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" + expectedresult: "PVC is Bound and the sha256 of the data file matches the pre-upgrade content" + position: 3 + custom_field: + "15": "TestLonghornPostUpgrade" + - title: "Longhorn chart upgrade after Rancher upgrade" + description: "Upgrade Longhorn to the version offered by the upgraded Rancher catalog and verify workload data" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Resolve the latest Longhorn chart version from the cluster-scoped catalog" + data: "Chart: longhorn from the rancher-charts repo" + expectedresult: "The latest cluster-scoped chart version is resolved" + position: 2 + - action: "Upgrade the Longhorn chart with retry" + data: "longhorn and longhorn-crd to the latest version" + expectedresult: "Both charts reach Deployed state and workloads become Ready" + position: 3 + - action: "Verify the chart version and the checksum workload" + data: "Chart version equals latest; PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" + expectedresult: "Installed chart version matches latest and the sha256 of the data file still matches the pre-upgrade content" + position: 4 + custom_field: + "15": "TestLonghornChartUpgrade" From f56386578fd546f95c31da2cfd5e4c9cf5dc624e Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 09:23:49 -0500 Subject: [PATCH 02/19] Add validation/Jenkinsfile.rancher-upgrade.e2e six-stage upgrade orchestrator Dedicated pipeline for the Rancher server upgrade validation (issue #968, PRD #946), community leg: deploy 2.14-latest env, pre-upgrade Longhorn phase, in-place server upgrade via make rancher-upgrade with rotated-token re-injection, post-upgrade and chart-upgrade phases against one shared Qase run, destroy-always with failure-only diagnostics. Versions resolve at runtime via make rancher-resolve-version when the params are empty. Structurally cloned from validation/Jenkinsfile.e2e and the airgap Jenkinsfile call shapes; runs go-pit-validation-job per phase with -runs TestLonghornUpgradeTestSuite/ selectors. --- validation/Jenkinsfile.rancher-upgrade.e2e | 981 +++++++++++++++++++++ 1 file changed, 981 insertions(+) create mode 100755 validation/Jenkinsfile.rancher-upgrade.e2e diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e new file mode 100755 index 0000000000..b867c1c195 --- /dev/null +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -0,0 +1,981 @@ +#!groovy +/** + * Dedicated Rancher server upgrade validation pipeline (rancher/tests#968, PRD #946). + * + * Six-stage flow, community leg: + * 1. Deploys a 2.14-latest environment via qa-infra-automation + * (make backend-s3, workspace-new, infra-up, cluster, rancher) + * 2. Pre-upgrade phase: installs Longhorn + seeds a checksum workload + * (validation/rancherupgrade, TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade) + * 3. Upgrades the Rancher server in place via `make rancher-upgrade` + * and re-injects the rotated admin token into the test config + * 4. Post-upgrade phase: asserts server version, Longhorn survival, workload data + * 5. Chart-upgrade phase: upgrades Longhorn to the version offered by the + * upgraded Rancher's catalog and re-verifies the workload data + * 6. Always destroys the environment (infra-down AUTO_APPROVE=yes) + * + * Versions resolve at runtime when RANCHER_VERSION_FROM / RANCHER_VERSION_TO_UPGRADE + * are empty (qa-infra-automation `make rancher-resolve-version`); pins override. + * + * Structurally cloned from validation/Jenkinsfile.e2e (deploy flow, admin-token + * injection, Qase testrun lifecycle, destroy post) and validation/pipeline/ + * Jenkinsfile.airgap-rke2-tests (make.runTarget call shapes, teardown guard). + * Unlike Jenkinsfile.e2e, this pipeline has no parallel downstream fan-out: the + * three test phases run sequentially against one shared Qase test run because + * each phase depends on the previous one's in-cluster state. + * + * Consumes shared functions from qa-jenkins-library: + * make.runTarget (backend-s3, workspace-new, infra-up, cluster, rancher, + * rancher-resolve-version, rancher-upgrade, infra-down) + * airgap.standardCheckout + * infrastructure.parseAndSubstituteVars, infrastructure.writeConfig, + * infrastructure.generateWorkspaceName, infrastructure.archiveWorkspaceName, + * infrastructure.writeSshKey, infrastructure.detectPublicIp + * property.useWithProperties + * + * Admin token generation uses a direct ansible-playbook call inside + * rancher-go-test:latest (no make token target exists upstream yet). + */ + +def libraryBranch = env.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +library "qa-jenkins-library@${libraryBranch}" + +def DEFAULT_RANCHER_PASSWORD = 'X%S3bPyg@n8kWcRs675l#fH0' + +// Suite/method selectors passed as GOTEST_TESTCASE to the downstream executor. +// Each phase is its own go test invocation (-run regex), so cross-phase state +// lives in-cluster under deterministic names, never in process memory. +def PRE_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade"' +def POST_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade"' +def CHART_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade"' + +/** + * Shared teardown helper: re-renders tfvars from params, tears down tofu workspace + * via make infra-down AUTO_APPROVE=yes. + * Sets env.INFRA_CLEANED to 'true' on success. + */ +def destroyInfrastructure(String reason) { + def wsName = '' + try { + if (fileExists('workspace_name.txt')) { + wsName = readFile('workspace_name.txt')?.trim() + } + } catch (e) { + echo "Could not read workspace_name.txt: ${e.message}" + } + + if (!wsName || !env.INFRA_DIR || env.INFRA_CLEANED == 'true') { + echo "Skipping teardown (${reason}): no workspace, no infra dir, or already cleaned" + return + } + + try { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + // Re-render terraform.tfvars so tofu destroy can access AWS state + def tofuDir = "${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes" + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME, + 'RUNNER_PUBLIC_IP': env.RUNNER_PUBLIC_IP + ] + ) + infrastructure.writeConfig( + path: "${tofuDir}/terraform.tfvars", + content: terraformConfig + ) + make.runTarget(target: 'infra-down', dir: env.INFRA_DIR, makeArgs: "WORKSPACE=${wsName} AUTO_APPROVE=yes") + env.INFRA_CLEANED = 'true' + } + } catch (cleanupErr) { + echo "Teardown failed (${reason}): ${cleanupErr.message}" + } +} + +/** + * Failure-only diagnostics: capture cluster state through the rancher-infra-tools + * container (kubectl + helm + kubeconfig from the deploy) into workspace files + * and archive them. Best-effort; never masks the failure it reports on. + */ +def collectUpgradeDiagnostics() { + if (!env.INFRA_DIR) { + echo 'Skipping diagnostics: no infra checkout available' + return + } + try { + def workspace = pwd() + sh """ + docker run --rm --platform linux/amd64 \ + -v ${workspace}:/workspace \ + -e KUBECONFIG=/workspace/${env.INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml \ + -w /workspace \ + rancher-infra-tools:latest \ + sh -c ' + set +e + kubectl get nodes -o wide > diagnostics-nodes.txt 2>&1 + kubectl get pods -A > diagnostics-pods.txt 2>&1 + helm history rancher -n cattle-system > diagnostics-helm-history.txt 2>&1 + kubectl logs -n cattle-system -l app=rancher --tail=300 > diagnostics-rancher-logs.txt 2>&1 + kubectl logs -n cattle-cluster-system -l app=cattle-cluster-agent --tail=300 > diagnostics-agent-logs.txt 2>&1 + exit 0 + ' + """ + if (params.REPORT_ARTIFACTS) { + archiveArtifacts artifacts: 'diagnostics-*.txt', allowEmptyArchive: true + } + } catch (diagErr) { + echo "Diagnostics collection failed (non-fatal): ${diagErr.message}" + } +} + +pipeline { + agent any + + options { + ansiColor('xterm') + // Deploy + three sequential test phases + server upgrade + destroy + timeout(time: 240, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '30')) + } + + parameters { + // ── Repository parameters ────────────────────────────── + string( + name: 'QA_JENKINS_LIBRARY_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-jenkins-library to use' + ) + string( + name: 'TESTS_REPO_URL', + defaultValue: 'https://github.com/rancher/tests', + description: 'URL of rancher/tests repository' + ) + string( + name: 'TESTS_BRANCH', + defaultValue: 'main', + description: 'Branch of rancher/tests repository' + ) + string( + name: 'QA_INFRA_REPO_URL', + defaultValue: 'https://github.com/rancher/qa-infra-automation', + description: 'URL of qa-infra-automation repository' + ) + string( + name: 'QA_INFRA_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-infra-automation repository' + ) + + // ── Upgrade parameters ───────────────────────────────── + string( + name: 'RANCHER_VERSION_FROM', + defaultValue: '', + description: 'Source Rancher version to deploy (empty = resolve latest 2.14.x patch at runtime)' + ) + string( + name: 'RANCHER_VERSION_TO_UPGRADE', + defaultValue: '', + description: 'Target Rancher version for the server upgrade (empty = resolve latest released final at runtime)' + ) + string( + name: 'RANCHER_CHART_REPO_FLAVOR', + defaultValue: 'community', + description: 'Chart repo flavor for the upgrade (community|prime; prime is a later slice)' + ) + + // ── Infrastructure parameters ────────────────────────── + string( + name: 'HOSTNAME_PREFIX', + defaultValue: 'pitupgrade', + description: 'Hostname prefix for *.qa.rancher.space and other AWS resources' + ) + string( + name: 'RKE2_VERSION', + defaultValue: 'v1.35.1+rke2r1', + description: 'RKE2 version to deploy' + ) + password( + name: 'RANCHER_BOOTSTRAP_PASSWORD', + defaultValue: DEFAULT_RANCHER_PASSWORD, + description: 'Rancher bootstrap password for initial setup' + ) + password( + name: 'RANCHER_ADMIN_PASSWORD', + defaultValue: DEFAULT_RANCHER_PASSWORD, + description: 'Rancher admin password after bootstrap' + ) + booleanParam( + name: 'DESTROY_ON_FAILURE', + defaultValue: true, + description: 'Collect diagnostics and tear down infrastructure if pipeline fails' + ) + booleanParam( + name: 'DESTROY_AFTER_TESTS', + defaultValue: true, + description: 'Tear down infrastructure after tests complete' + ) + + // ── Terraform parameters ─────────────────────────────── + string( + name: 'S3_BUCKET_NAME', + defaultValue: 'jenkins-terraform-state-storage', + description: 'S3 bucket name where Terraform state is stored' + ) + string( + name: 'S3_BUCKET_REGION', + defaultValue: 'us-east-2', + description: 'AWS region where the S3 bucket is located' + ) + string( + name: 'S3_KEY_PREFIX', + defaultValue: 'terraform.tfstate', + description: 'S3 key prefix for the Terraform state files' + ) + text( + name: 'TERRAFORM_CONFIG', + description: 'Terraform tfvars for the cluster_nodes module (non-airgap)', + defaultValue: '''aws_access_key = "${AWS_ACCESS_KEY_ID}" +aws_secret_key = "${AWS_SECRET_ACCESS_KEY}" +aws_ami = "ami-0600e22b69546c15c" +instance_type = "t3a.2xlarge" +aws_vpc = "vpc-bfccf4d7" +aws_subnet = "subnet-ee8cac86" +ephemeral_sg_ingress_cidrs = ["${RUNNER_PUBLIC_IP}/32"] +airgap_setup = false +proxy_setup = false +aws_volume_size = 500 +aws_volume_type = "gp3" +aws_hostname_prefix = "${HOSTNAME_PREFIX}" +aws_region = "us-east-2" +aws_route53_zone = "qa.rancher.space" +aws_ssh_user = "ubuntu" +public_ssh_key = "ssh_public_key.pub" +private_ssh_key = "~/.ssh/${AWS_SSH_PEM_KEY_NAME}" +nodes = [ + { + count = 3 + role = ["etcd", "cp", "worker"] + }, + { + count = 1 + role = ["worker"] + } +]''' + ) + text( + name: 'ANSIBLE_VARIABLES', + description: 'Ansible group_vars for RKE2 install + Rancher Helm deploy (non-airgap). Rancher version keys resolve from the pre-upgrade (FROM) version; rancher_version_upgrade keys hold the resolved target the server is upgraded to via make rancher-upgrade.', + defaultValue: '''--- +kubernetes_version: ${RKE2_VERSION} +cni: calico +kubeconfig_file: "/workspace/qa-infra-automation/ansible/rke2/default/kubeconfig.yaml" +cert_manager_version: "1.20.0" +bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +password: "${RANCHER_ADMIN_PASSWORD}" +rancher_version: ${RANCHER_VERSION} +rancher_image_tag: ${RANCHER_IMAGE_TAG} +rancher_chart_repo_url: "${RANCHER_CHART_REPO_URL}" +upgrade_mode: false +rancher_version_upgrade: "${RANCHER_VERSION_UPGRADE}" +rancher_image_tag_upgrade: "${RANCHER_IMAGE_TAG_UPGRADE}" +rancher_chart_repo_upgrade: "${RANCHER_UPGRADE_REPO_NAME}" +k8s_upgrade_mode: false +server_flags: | + ingress-controller: traefik''' + ) + + // ── Test parameters ──────────────────────────────────── + text( + name: 'CATTLE_TEST_CONFIG', + description: 'Test configuration YAML (cattle-config). rancher.adminToken is injected at runtime (initial + rotated after the server upgrade); rancherUpgradeInput.targetVersion is substituted from the resolved upgrade target.', + defaultValue: '''rancher: + host: ${HOSTNAME_PREFIX}.qa.rancher.space + adminPassword: ${ADMIN_PASSWORD} + insecure: true + # Deliberately false: pre-upgrade resources (Longhorn chart, PVC, workload) + # must survive into the post-upgrade phases; the pipeline environment + # teardown is the cleanup. + cleanup: false + # The qa-infra-automation cluster_nodes RKE2 cluster is Rancher's own + # ("local") cluster; the upgrade suite installs Longhorn on it. + clusterName: local +rancherUpgradeInput: + targetVersion: "${RANCHER_VERSION_UPGRADE}"''', + ) + booleanParam( + name: 'REPORT_ARTIFACTS', + defaultValue: true, + description: 'Archive token metadata, Qase run config, and failure diagnostics as build artifacts' + ) + + // ── Downstream job parameters ───────────────────────── + string( + name: 'INDIVIDUAL_JOB', + defaultValue: 'go-pit-validation-job', + description: 'Name of the downstream Jenkins job that runs individual Go test suites' + ) + string( + name: 'TEST_PACKAGE', + defaultValue: './validation/rancherupgrade/...', + description: 'Go test package path passed to the downstream job' + ) + string( + name: 'TEST_TIMEOUT', + defaultValue: '60m', + description: 'Timeout for downstream test jobs' + ) + string( + name: 'QASE_REPORTER_SCRIPT', + defaultValue: 'build_qase_reporter_v2.sh', + description: 'Qase reporter script name in pipeline/scripts/' + ) + string( + name: 'QASE_SCHEMA_PREFIX', + defaultValue: 'pit', + description: 'Schema file prefix for Qase operations, for example hostbusters or pit' + ) + } + + environment { + WORKSPACE_NAME = '' + INFRA_CLEANED = 'false' + YQ_VERSION = 'v4.47.2' + // Folder-injected TEST_RUN_NAME (if present) wins; the fallback keeps + // the shared Qase run name readable when it is not injected. + TEST_RUN_NAME = "${env.TEST_RUN_NAME ?: 'Rancher Server Upgrade Validation'}" + } + + stages { + stage('Checkout') { + steps { + script { + def dirs = airgap.standardCheckout( + testsRepo: [url: params.TESTS_REPO_URL, branch: params.TESTS_BRANCH], + infraRepo: [url: params.QA_INFRA_REPO_URL, branch: params.QA_INFRA_BRANCH] + ) + env.TESTS_DIR = dirs.testsDir + env.INFRA_DIR = dirs.infraDir + + def hostnamePrefix = params.HOSTNAME_PREFIX ?: 'unknown' + currentBuild.displayName = "#${env.BUILD_NUMBER} - ${hostnamePrefix}" + currentBuild.description = "Rancher server upgrade validation for ${hostnamePrefix}" + } + } + } + + stage('Build Docker Images') { + steps { + // rancher-infra-tools:latest — used by make.groovy / infra.* for all make targets + sh "docker build --no-cache --platform linux/amd64 -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." + // rancher-go-test:latest — used for qase binary build and admin-token playbook + sh "docker build --no-cache --platform linux/amd64 -t rancher-go-test:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.airgap-go-tests ." + } + } + + stage('Setup Tools') { + steps { + sh """ + wget -qO ./yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + wget -qO ./yq_checksums https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/checksums + grep -q '^yq_linux_amd64 ' ./yq_checksums + awk '/^yq_linux_amd64 / {print \$19 \" ./yq\"}' ./yq_checksums | sha256sum -c - + rm -f ./yq_checksums + chmod +x ./yq + """ + } + } + + // Auto-detect this runner's public IP so the ephemeral security + // group's SSH/LB ingress rule (ephemeral_sg_ingress_cidrs) allows + // the IP that Ansible/SSH will actually originate from. + stage('Detect Public IP') { + steps { + script { + env.RUNNER_PUBLIC_IP = infrastructure.detectPublicIp() + } + } + } + + // ── Resolve the upgrade window ───────────────────────── + // Empty params resolve at runtime via qa-infra-automation's + // rancher-resolve-version target; explicit pins pass through. + // The chosen lines/regexes and resolved versions are echoed so CI + // logs show exactly which version each stage ran against. + stage('Resolve Rancher Versions') { + steps { + script { + env.RANCHER_FROM_VERSION = params.RANCHER_VERSION_FROM?.trim() ?: '' + env.RANCHER_TO_VERSION = params.RANCHER_VERSION_TO_UPGRADE?.trim() ?: '' + + if (!env.RANCHER_FROM_VERSION) { + echo 'RANCHER_VERSION_FROM empty — resolving latest 2.14.x patch (SOURCE_LINE=2.14, latest final of the line)' + def resolveOut = make.runTarget( + target: 'rancher-resolve-version', + dir: env.INFRA_DIR, + makeArgs: ['SOURCE_LINE=2.14'], + returnStdout: true, + passAwsCreds: false + ) + writeFile file: 'resolve-from.json', text: resolveOut + env.RANCHER_FROM_VERSION = sh( + script: "./yq '.chart_version' resolve-from.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + env.RANCHER_FROM_IMAGE_TAG = sh( + script: "./yq '.image_tag' resolve-from.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + } else { + env.RANCHER_FROM_IMAGE_TAG = env.RANCHER_FROM_VERSION.startsWith('v') ? env.RANCHER_FROM_VERSION : "v${env.RANCHER_FROM_VERSION}" + echo "RANCHER_VERSION_FROM pinned: ${env.RANCHER_FROM_VERSION} (image tag ${env.RANCHER_FROM_IMAGE_TAG})" + } + + if (!env.RANCHER_TO_VERSION) { + echo 'RANCHER_VERSION_TO_UPGRADE empty — resolving latest released final (no SOURCE_LINE, repo index latest final)' + def resolveOut = make.runTarget( + target: 'rancher-resolve-version', + dir: env.INFRA_DIR, + returnStdout: true, + passAwsCreds: false + ) + writeFile file: 'resolve-to.json', text: resolveOut + env.RANCHER_TO_VERSION = sh( + script: "./yq '.chart_version' resolve-to.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + env.RANCHER_TO_IMAGE_TAG = sh( + script: "./yq '.image_tag' resolve-to.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + } else { + env.RANCHER_TO_IMAGE_TAG = env.RANCHER_TO_VERSION.startsWith('v') ? env.RANCHER_TO_VERSION : "v${env.RANCHER_TO_VERSION}" + echo "RANCHER_VERSION_TO_UPGRADE pinned: ${env.RANCHER_TO_VERSION} (image tag ${env.RANCHER_TO_IMAGE_TAG})" + } + + echo """ + Upgrade window resolved: + FROM (deploy): ${env.RANCHER_FROM_VERSION} (${env.RANCHER_FROM_IMAGE_TAG}) + TO (upgrade): ${env.RANCHER_TO_VERSION} (${env.RANCHER_TO_IMAGE_TAG}) + Chart flavor: ${params.RANCHER_CHART_REPO_FLAVOR} + """.stripIndent() + currentBuild.description = "Upgrade ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (${params.RANCHER_CHART_REPO_FLAVOR}) | ${params.HOSTNAME_PREFIX}" + } + } + } + + stage('Deploy Rancher Pre-Upgrade Environment') { + steps { + script { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/default" + def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes" + def workspaceName = '' + + // ── SSH key setup ────────────────────────── + stage('Configure SSH Key') { + infrastructure.writeSshKey( + keyContent: env.AWS_SSH_PEM_KEY, + keyName: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ) + // Copy public key into the tofu module directory so + // tofu's file() function can read it (restricted to + // the module directory tree). + sh "cp .ssh/*.pub ${tofuModulePath}/ssh_public_key.pub" + } + + // ── Render terraform.tfvars ───────────────── + // Must exist before any make target runs (check-tofu-dir validates it). + stage('Configure Tofu Variables') { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME, + 'RUNNER_PUBLIC_IP': env.RUNNER_PUBLIC_IP + ] + ) + infrastructure.writeConfig( + path: "${tofuModulePath}/terraform.tfvars", + content: terraformConfig + ) + } + + // ── Initialize S3 backend ─────────────────── + stage('Initialize Tofu Backend') { + make.runTarget( + target: 'backend-s3', + dir: env.INFRA_DIR, + makeArgs: [ + "BUCKET=${params.S3_BUCKET_NAME}", + "KEY=${params.S3_KEY_PREFIX}", + "REGION=${params.S3_BUCKET_REGION}" + ] + ) + } + + // ── Create tofu workspace ─────────────────── + stage('Create Workspace') { + workspaceName = infrastructure.generateWorkspaceName( + prefix: 'jenkins_rancher_upgrade_workspace', + suffix: params.HOSTNAME_PREFIX, + includeTimestamp: false + ) + make.runTarget( + target: 'workspace-new', + dir: env.INFRA_DIR, + makeArgs: "WORKSPACE=${workspaceName}" + ) + infrastructure.archiveWorkspaceName(workspaceName: workspaceName) + env.WORKSPACE_NAME = workspaceName + } + + // ── Provision infra + generate inventory ──── + stage('Provision Infrastructure') { + make.runTarget( + target: 'infra-up', + dir: env.INFRA_DIR, + makeArgs: "WORKSPACE=${workspaceName}" + ) + } + + // ── Write group_vars/all.yml ──────────────── + // Must exist before make cluster (make infra-up generates + // inventory.yml but not the group_vars subdirectory). + // Rancher version keys carry the resolved FROM version; + // the *_upgrade keys carry the resolved TO version the + // later `make rancher-upgrade` stage targets. + stage('Configure Ansible Variables') { + sh "mkdir -p ${ansiblePath}/inventory/group_vars" + + // The FROM version is a stable semver patch line, so + // its chart repo is the stable "latest" index. + def fromChartRepoUrl = 'https://releases.rancher.com/server-charts/latest' + def upgradeRepoName = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'rancher-prime-upgrade' : 'rancher-upgrade' + + def ansibleVars = infrastructure.parseAndSubstituteVars( + content: params.ANSIBLE_VARIABLES, + envVars: [ + 'RKE2_VERSION': params.RKE2_VERSION, + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_VERSION': env.RANCHER_FROM_VERSION, + 'RANCHER_IMAGE_TAG': env.RANCHER_FROM_IMAGE_TAG, + 'RANCHER_CHART_REPO_URL': fromChartRepoUrl, + 'RANCHER_VERSION_UPGRADE': env.RANCHER_TO_VERSION, + 'RANCHER_IMAGE_TAG_UPGRADE': env.RANCHER_TO_IMAGE_TAG, + 'RANCHER_UPGRADE_REPO_NAME': upgradeRepoName + ] + ) + + def rancherVars = infrastructure.parseAndSubstituteVars( + content: '''deploy_rancher: true +install_helm: true +rancher_hostname: "${HOSTNAME_PREFIX}.qa.rancher.space" +rancher_bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +rancher_admin_password: "${RANCHER_ADMIN_PASSWORD}" +rancher_use_bundled_system_charts: true +''', + envVars: [ + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD + ] + ) + + // Merge ANSIBLE_VARIABLES + Rancher vars into a single group_vars/all.yml + infrastructure.writeConfig( + path: "${ansiblePath}/inventory/group_vars/all.yml", + content: ansibleVars + '\n' + rancherVars + ) + } + + // ── Deploy RKE2 cluster (ansible; no AWS creds) ── + stage('Deploy RKE2 Cluster') { + make.runTarget( + target: 'cluster', + dir: env.INFRA_DIR, + passAwsCreds: false + ) + } + + // ── Deploy pre-upgrade Rancher (helm; no AWS creds) ── + stage('Deploy Rancher') { + make.runTarget( + target: 'rancher', + dir: env.INFRA_DIR, + passAwsCreds: false + ) + } + + // ── Capture Rancher hostname ──────────────── + stage('Output Infrastructure Details') { + // Rancher hostname is deterministically constructed from HOSTNAME_PREFIX + // matching the value set in rancherVars (rancher_hostname). + def rancherHostname = "${params.HOSTNAME_PREFIX}.qa.rancher.space" + echo "Rancher Hostname: https://${rancherHostname}" + env.RANCHER_HOSTNAME = rancherHostname + } + } + } + } + } + + stage('Create Test Config and Inject Admin Token') { + steps { + script { + def workspace = pwd() + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + + // Create cattle-config with variable substitution, including + // the resolved upgrade target for rancherUpgradeInput.targetVersion. + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY' + ]) { + def resolvedConfig = infrastructure.parseAndSubstituteVars( + content: params.CATTLE_TEST_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_VERSION_UPGRADE': env.RANCHER_TO_VERSION + ] + ) + writeFile file: cattleConfigPath, text: resolvedConfig.trim() + '\n' + } + + // Generate admin token via Ansible playbook + // TODO: replace with 'make token' once that target is added upstream + def containerConfigPath = "/workspace/${cattleConfigPath}" + def tokenTtl = env.RANCHER_TOKEN_TTL ?: '0' + def tokenDescription = "jenkins-rancher-upgrade-${env.BUILD_NUMBER}" + + // Diagnostic logging to validate env + output path before token generation + echo "Token generation context: workspace=${workspace}, config=${containerConfigPath}" + echo "RANCHER_ADMIN_PASSWORD present in params: ${params.RANCHER_ADMIN_PASSWORD ? 'yes' : 'no'}" + + sh """ + docker run --rm --platform linux/amd64 \ + --name generate-token \ + -e RANCHER_ADMIN_PASSWORD \ + -e RANCHER_URL=https://${params.HOSTNAME_PREFIX}.qa.rancher.space \ + -e ANSIBLE_CONFIG=/workspace/${env.INFRA_DIR}/ansible/ansible.cfg \ + -v ${workspace}:/workspace \ + -w /workspace/${env.INFRA_DIR}/ansible/rke2/default \ + rancher-go-test:latest \ + ansible-playbook -i inventory/inventory.yml /workspace/${env.INFRA_DIR}/ansible/rancher/token/generate-admin-token.yml \ + -e rancher_cattle_config_file=${containerConfigPath} \ + -e rancher_token_ttl=${tokenTtl} \ + -e rancher_token_description=${tokenDescription} \ + -e rancher_token_output_format=json \ + -e rancher_token_output_file=/workspace/rancher-token.json \ + -e rancher_url=https://${params.HOSTNAME_PREFIX}.qa.rancher.space + """ + + def tokenFileExists = fileExists('rancher-token.json') + echo "Token file present after generation: ${tokenFileExists}" + + if (tokenFileExists) { + // Redact secret fields from token JSON via Docker container. + // Pipeline Utility Steps plugin (readJSON/writeJSON) is not installed, + // and python3 is not available on the Jenkins agent host. + sh "docker run --rm -v \$(pwd):/workspace -w /workspace rancher-go-test:latest python3 -c \"import json; d=json.load(open('rancher-token.json')); [d.pop(k,None) for k in ('token','bearerToken','accessKey','secretKey','value')]; json.dump(d,open('rancher-token-metadata.json','w'),indent=2)\"" + if (params.REPORT_ARTIFACTS) { + archiveArtifacts artifacts: 'rancher-token-metadata.json' + echo 'Archived redacted token metadata: rancher-token-metadata.json' + } + + // Infrastructure, Rancher deploy, and admin token all succeeded. + // Shared library helpers (infra.*) may have set currentBuild.result to + // UNSTABLE/FAILURE via internal catchError() wrappers for non-fatal warnings, + // which would cause declarative pipeline to skip downstream test stages. + // Reset build result to SUCCESS so the upgrade phases can execute. + if (currentBuild.result && currentBuild.result != 'SUCCESS') { + echo "Resetting currentBuild.result from '${currentBuild.result}' to 'SUCCESS' — token generation succeeded, allowing upgrade phases to run" + currentBuild.result = 'SUCCESS' + } + } else { + error 'rancher-token.json not found after token generation — cannot run test phases without an admin token' + } + } + } + } + + // ── Phase 1: pre-upgrade Longhorn install + workload seed ── + // Starts the shared Qase run once; all three phases report into it. + stage('Pre-Upgrade Longhorn Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def workspace = pwd() + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + def testRunConfig = "${workspace}/testrunconfig.yaml" + def qaseBinary = "${workspace}/testrun" + + // Build qase testrun binary into the workspace root so the + // agent-side invocation below and testrunconfig.yaml (written + // to the binary's CWD) line up. + sh """ + docker run --rm --platform linux/amd64 \ + -v ${workspace}:/workspace \ + -w /workspace/${env.TESTS_DIR} \ + rancher-go-test:latest \ + sh -c 'env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /workspace/testrun ./validation/pipeline/qase/testrun' + """ + + // Start the shared qase test run (completed after the final phase) + sh "${qaseBinary} --startRun=true" + + // Read test run ID + def testRunID = '' + if (fileExists(testRunConfig)) { + testRunID = sh( + script: "./yq '.id' ${testRunConfig}", + returnStdout: true + ).trim() + } + echo "Shared Qase test run started: id=${testRunID}" + if (params.REPORT_ARTIFACTS && fileExists(testRunConfig)) { + archiveArtifacts artifacts: 'testrunconfig.yaml', allowEmptyArchive: true + } + + // NOTE: unlike Jenkinsfile.e2e, rancher.cleanup is NOT set to + // true and no desiredflags are forced here — the upgrade suite's + // cross-phase state must survive, and the pipeline environment + // teardown is the cleanup. + def configYaml = readFile(file: cattleConfigPath) + + echo "Dispatching pre-upgrade phase: GOTEST_TESTCASE ${PRE_UPGRADE_TESTCASE}" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: PRE_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } + } + } + } + + // ── Phase 2: in-place Rancher server upgrade ── + // `make rancher-upgrade` rotates the Rancher admin token and writes the + // new one to ansible/rancher/default-ha/generated.tfvars; the old token + // is revoked, so the test config MUST be re-injected before the + // post-upgrade phases dispatch (verified live: old token 401s). + stage('Upgrade Rancher Server') { + steps { + script { + echo "Upgrading Rancher server: ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (flavor ${params.RANCHER_CHART_REPO_FLAVOR})" + make.runTarget( + target: 'rancher-upgrade', + dir: env.INFRA_DIR, + makeArgs: [ + "ENV=default", + "RANCHER_VERSION_TO_UPGRADE=${env.RANCHER_TO_VERSION}", + "RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}" + ], + passAwsCreds: false + ) + + // Re-inject the rotated admin token into the test config + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + def generatedTfvars = "${env.INFRA_DIR}/ansible/rancher/default-ha/generated.tfvars" + + if (!fileExists(generatedTfvars)) { + error "Rotated token file not found at ${generatedTfvars} — post-upgrade phases cannot authenticate" + } + def rotatedToken = sh( + script: "awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}", + returnStdout: true + ).trim() + if (!rotatedToken) { + error "Could not extract api_key from ${generatedTfvars} — post-upgrade phases cannot authenticate" + } + echo "Rotated admin token extracted (length ${rotatedToken.length()}); re-injecting into test config" + + // The token never leaves this shell step (no Groovy/env + // interpolation), so it cannot leak into the build log. + sh """ + token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) + ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} + """ + echo "Test config now carries the post-upgrade admin token; rancherUpgradeInput.targetVersion=${env.RANCHER_TO_VERSION}" + } + } + } + + // ── Phase 3: post-upgrade assertions ── + stage('Post-Upgrade Longhorn Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def testRunID = sh( + script: "./yq '.id' testrunconfig.yaml", + returnStdout: true + ).trim() + def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + + echo "Dispatching post-upgrade phase: GOTEST_TESTCASE ${POST_UPGRADE_TESTCASE} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: POST_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } + } + } + } + + // ── Phase 4: Longhorn chart upgrade through the upgraded catalog ── + // Completes the shared Qase run after this final phase; do not rely on + // the downstream job's own QASE_TEST_RUN_COMPLETE for the shared run. + stage('Longhorn Chart Upgrade Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def testRunID = sh( + script: "./yq '.id' testrunconfig.yaml", + returnStdout: true + ).trim() + def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + + echo "Dispatching chart-upgrade phase: GOTEST_TESTCASE ${CHART_UPGRADE_TESTCASE} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: CHART_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + + // Complete the shared qase test run + sh "./testrun --startRun=false" + env.QASE_RUN_COMPLETED = 'true' + echo "Shared Qase test run ${testRunID} completed" + } + } + } + } + } + + post { + failure { + script { + if (params.DESTROY_ON_FAILURE) { + collectUpgradeDiagnostics() + destroyInfrastructure('DESTROY_ON_FAILURE') + } + } + } + + always { + script { + // Complete the shared Qase run when a phase failed before the + // final stage could close it (no-op once already completed). + if (env.QASE_RUN_COMPLETED != 'true' && fileExists('testrun') && fileExists('testrunconfig.yaml')) { + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + sh './testrun --startRun=false' + } + env.QASE_RUN_COMPLETED = 'true' + } + } + + if (params.DESTROY_AFTER_TESTS) { + destroyInfrastructure('DESTROY_AFTER_TESTS') + } + + // Docker cleanup + sh 'docker rmi -f rancher-go-test:latest rancher-infra-tools:latest || true' + + // Sensitive artifact cleanup — shred or remove credentials from workspace + sh ''' + set +e + + cleanup_file() { + if [ -f "$1" ]; then + if command -v shred >/dev/null 2>&1; then + shred -u "$1" || rm -f "$1" + else + rm -f "$1" + fi + fi + } + + cleanup_dir() { + if [ -d "$1" ]; then + find "$1" -type f -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + rm -rf "$1" + fi + } + + cleanup_file ".go_test_env" + cleanup_file "rancher-token.json" + cleanup_file "rancher-token-metadata.json" + cleanup_file "workspace_name.txt" + cleanup_file "resolve-from.json" + cleanup_file "resolve-to.json" + + if [ -n "${INFRA_DIR}" ] && [ -d "${INFRA_DIR}" ]; then + find "${INFRA_DIR}" -type f \\( -name 'terraform.tfvars' -o -name 'generated.tfvars' -o -name '*.pem' -o -name 'id_rsa' -o -name 'id_ed25519' \\) -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + fi + + cleanup_dir ".ssh" + ''' + } + } + } +} From 3e14ff9192b91826a0eaf132e66c73063a085cbc Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 10:38:56 -0500 Subject: [PATCH 03/19] Add awsNodeTemplate to upgrade cattle-config for SG authorization The per-phase executor job can land on a different Jenkins agent than the orchestrator (observed in the first rancher-upgrade run: default-n367q vs default-xj00w). The LB ingress on the tofu-created ephemeral SG only allows the orchestrator's detected public IP, so a phase on another agent could not reach the Rancher API and rancher.NewClient timed out after 22 minutes of retries. awsNodeTemplate carries the three keys the executor's Authorize Security Group stage reads (securityGroup, vpcId, region) so it authorizes its own runner IP on tf--sg before dispatching to the API, matching the shape Jenkinsfile.e2e ships for the same reason. --- validation/Jenkinsfile.rancher-upgrade.e2e | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index b867c1c195..8ac1006043 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -295,7 +295,17 @@ server_flags: | text( name: 'CATTLE_TEST_CONFIG', description: 'Test configuration YAML (cattle-config). rancher.adminToken is injected at runtime (initial + rotated after the server upgrade); rancherUpgradeInput.targetVersion is substituted from the resolved upgrade target.', - defaultValue: '''rancher: + defaultValue: '''# The downstream per-phase executor can land on a different Jenkins agent +# than this orchestrator; awsNodeTemplate carries the three keys its +# Authorize Security Group stage needs (sg name, vpc, region) so the +# runner's public IP gets authorized on the tofu-created ephemeral SG +# (tf--sg) before the phases talk to the Rancher LB. +awsNodeTemplate: + region: us-east-2 + vpcId: vpc-bfccf4d7 + securityGroup: + - tf-${HOSTNAME_PREFIX}-sg +rancher: host: ${HOSTNAME_PREFIX}.qa.rancher.space adminPassword: ${ADMIN_PASSWORD} insecure: true From 90381bdcbb5bc4f28253f16f1021d617fb05ff2c Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 11:41:09 -0500 Subject: [PATCH 04/19] Make repo/executor params sticky against declarative parameter sync A declarative pipeline re-applies its parameters{} block to the job definition after each run, which silently overwrote the JJB-configured staging values (fork repos + slice branches) with the hard-coded main defaults on the mower job: the first run deployed from the fork, the next one fell back to upstream main and lost make rancher-resolve-version ('No rule to make target'). Defaulting these five params to their current value when set keeps the per-controller JJB configuration authoritative instead of racing it. --- validation/Jenkinsfile.rancher-upgrade.e2e | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 8ac1006043..80a19180be 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -152,24 +152,29 @@ pipeline { defaultValue: 'main', description: 'Branch of qa-jenkins-library to use' ) + // Repo params keep whatever the job was last triggered with: a + // declarative pipeline re-applies its parameters{} block to the job + // after each run, and hard-coded defaults here would clobber the + // JJB-configured per-controller values (fork + slice branches on the + // staging controller, upstream main on the main controller). string( name: 'TESTS_REPO_URL', - defaultValue: 'https://github.com/rancher/tests', + defaultValue: params.TESTS_REPO_URL ?: 'https://github.com/rancher/tests', description: 'URL of rancher/tests repository' ) string( name: 'TESTS_BRANCH', - defaultValue: 'main', + defaultValue: params.TESTS_BRANCH ?: 'main', description: 'Branch of rancher/tests repository' ) string( name: 'QA_INFRA_REPO_URL', - defaultValue: 'https://github.com/rancher/qa-infra-automation', + defaultValue: params.QA_INFRA_REPO_URL ?: 'https://github.com/rancher/qa-infra-automation', description: 'URL of qa-infra-automation repository' ) string( name: 'QA_INFRA_BRANCH', - defaultValue: 'main', + defaultValue: params.QA_INFRA_BRANCH ?: 'main', description: 'Branch of qa-infra-automation repository' ) @@ -328,7 +333,7 @@ rancherUpgradeInput: // ── Downstream job parameters ───────────────────────── string( name: 'INDIVIDUAL_JOB', - defaultValue: 'go-pit-validation-job', + defaultValue: params.INDIVIDUAL_JOB ?: 'go-pit-validation-job', description: 'Name of the downstream Jenkins job that runs individual Go test suites' ) string( From 263293c65c92f8f408c0a34755e89dc778d9873e Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 12:26:04 -0500 Subject: [PATCH 05/19] Wait up to ten minutes for the checksum PVC to bind The one-minute bind wait raced a freshly installed Longhorn: the first volume's provisioning (engine-image pull, instance-manager startup, CSI registration) can exceed a minute on a cold cluster, which failed the pre-upgrade phase with 'context deadline exceeded' in the mower run (the same step passed when the volume bound faster). Ten minutes matches the bound used for the rest of the storage waits. --- actions/storage/checksum.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go index 82ab4d305c..9f3d173cf7 100644 --- a/actions/storage/checksum.go +++ b/actions/storage/checksum.go @@ -66,7 +66,10 @@ func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageCla } pvc := &corev1.PersistentVolumeClaim{} - err = wait.PollUntilContextTimeout(context.Background(), pollInterval, defaults.OneMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + // TenMinuteTimeout: the first volume on a freshly installed Longhorn waits + // on engine-image pull, instance-manager startup, and CSI registration, + // which routinely exceeds a minute on a cold cluster. + err = wait.PollUntilContextTimeout(context.Background(), pollInterval, defaults.TenMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { pvc, err = wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) if err != nil { return false, err From bd78700ce424faaa65add61729f9b5a79a793433 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 12:51:36 -0500 Subject: [PATCH 06/19] Collect failure diagnostics before destroy and image cleanup Jenkins evaluates declarative post conditions in a fixed order with always before failure, so the failure-block diagnostics ran after the always block had already run infra-down and docker rmi: kubectl/helm had no rancher-infra-tools image left to run in (pull access denied, exit 125) and the cluster was already gone regardless. Diagnostics now run first inside always, gated on a non-SUCCESS result, while the images and the live cluster still exist; the failure block keeps only the DESTROY_ON_FAILURE fallback destroy. --- validation/Jenkinsfile.rancher-upgrade.e2e | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 80a19180be..0993fde1ee 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -914,7 +914,6 @@ rancher_use_bundled_system_charts: true failure { script { if (params.DESTROY_ON_FAILURE) { - collectUpgradeDiagnostics() destroyInfrastructure('DESTROY_ON_FAILURE') } } @@ -922,6 +921,15 @@ rancher_use_bundled_system_charts: true always { script { + // Failure diagnostics run here, not in the failure block: + // Jenkins evaluates post conditions in a fixed order with + // always first, so anything in failure{} would already have + // lost the docker images (rmi below) and the live cluster + // (destroy below). Collect while both still exist. + if (currentBuild.currentResult != 'SUCCESS') { + collectUpgradeDiagnostics() + } + // Complete the shared Qase run when a phase failed before the // final stage could close it (no-op once already completed). if (env.QASE_RUN_COMPLETED != 'true' && fileExists('testrun') && fileExists('testrunconfig.yaml')) { From c1237a56ac5fb2c00618e91eb11b790e657789aa Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 13:52:40 -0500 Subject: [PATCH 07/19] Smoke-check the rotated token and the dispatched config post-upgrade The post-upgrade phase failed with an instant 401 from rancher.NewClient even though the rotated token had been extracted (length 66) and yq-injected into the dispatched config. Two fail-fast checks now pinpoint which side is broken: the injected token must get HTTP 200 from the upgraded Rancher API before any phase is dispatched, and the config about to be dispatched must byte-match the token in generated.tfvars. Both run under set +x so the token value stays out of the xtrace (the previous run echoed it into the build log). --- validation/Jenkinsfile.rancher-upgrade.e2e | 24 +++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 0993fde1ee..2a8f4fcf5c 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -832,10 +832,19 @@ rancher_use_bundled_system_charts: true echo "Rotated admin token extracted (length ${rotatedToken.length()}); re-injecting into test config" // The token never leaves this shell step (no Groovy/env - // interpolation), so it cannot leak into the build log. + // interpolation, set +x keeps it out of the xtrace), + // and the injected token is smoke-checked against the + // live API before any phase is dispatched on it. sh """ + set +x token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} + status=\$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) + if [ "\$status" != "200" ]; then + echo "Rotated token rejected by the upgraded Rancher API: HTTP \$status (token length \${#token})" + exit 1 + fi + echo "Rotated token smoke-check: HTTP 200 from the upgraded Rancher API" """ echo "Test config now carries the post-upgrade admin token; rancherUpgradeInput.targetVersion=${env.RANCHER_TO_VERSION}" } @@ -853,6 +862,19 @@ rancher_use_bundled_system_charts: true ).trim() def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + // Fail fast (without printing the token) if the config + // about to be dispatched does not carry the rotated one. + sh """ + set +x + expected=\$(awk -F'\"' '/^api_key/{print \$2}' ${env.INFRA_DIR}/ansible/rancher/default-ha/generated.tfvars) + actual=\$(./yq '.rancher.adminToken' ${env.TESTS_DIR}/cattle-config.yaml) + if [ "\$expected" != "\$actual" ]; then + echo "Dispatched config adminToken does not match the rotated token (expected length \${#expected}, got length \${#actual})" + exit 1 + fi + echo "Dispatched config carries the rotated admin token (length \${#expected})" + """ + echo "Dispatching post-upgrade phase: GOTEST_TESTCASE ${POST_UPGRADE_TESTCASE} (Qase run ${testRunID})" build job: params.INDIVIDUAL_JOB, parameters: [ string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), From a5420c2f09e4e63c5622301d509547466ba692d3 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 14:47:21 -0500 Subject: [PATCH 08/19] Accept the insecure LB cert in the rotated-token smoke check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The smoke-check curl died with exit 60 (TLS verification) against pitupgrade.qa.rancher.space before it ever reached auth — the same endpoint the tests already call with insecure: true. Pass -k so the check actually adjudicates the token. --- validation/Jenkinsfile.rancher-upgrade.e2e | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 2a8f4fcf5c..cb0921dd84 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -839,7 +839,7 @@ rancher_use_bundled_system_charts: true set +x token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} - status=\$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) + status=\$(curl -sk -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) if [ "\$status" != "200" ]; then echo "Rotated token rejected by the upgraded Rancher API: HTTP \$status (token length \${#token})" exit 1 From ae95642df430129804c4cb315b2998686efa448c Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 15:38:02 -0500 Subject: [PATCH 09/19] Export the token before yq injection (empty-adminToken 401 root cause) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit token=$(awk ...) creates an unexported shell variable, but yq's strenv() reads the process environment, so the injection silently wrote an empty string into rancher.adminToken — the post-upgrade phase then authenticated with an empty bearer token (instant 401). curl in the same step still worked because its header uses shell expansion, which is why the smoke-check passed while the config check failed (expected length 66, got length 0). export the variable so the child yq process can see it. --- validation/Jenkinsfile.rancher-upgrade.e2e | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index cb0921dd84..9b6e5f4d9d 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -838,6 +838,9 @@ rancher_use_bundled_system_charts: true sh """ set +x token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) + # export: yq's strenv() reads the environment, not the + # shell's unexported variables (empty-token 401 root cause). + export token ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} status=\$(curl -sk -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) if [ "\$status" != "200" ]; then From 739d5acd258790135813d988daa9f83f3618dd0e Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 8 Oct 2026 17:26:09 -0500 Subject: [PATCH 10/19] Address PR review: phase progress logs, build-pinned image tags - logrus milestone logs in each upgrade phase (version resolution, chart install/upgrade, workload seed, checksum verification) so the job log shows where each phase is without reading the assertions backwards. - rancher-infra-tools/rancher-go-test now carry a build-scoped tag (pitupgrade-) used by every direct docker invocation in this pipeline, so this job no longer depends on whatever another job last built or removed under the shared :latest tag on the same agent; the :latest alias is still applied for qa-jenkins-library's make.runTarget image resolution. --- validation/Jenkinsfile.rancher-upgrade.e2e | 33 ++++++++++++++-------- validation/rancherupgrade/longhorn_test.go | 13 +++++++++ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 9b6e5f4d9d..90be3147cc 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -33,8 +33,8 @@ * infrastructure.writeSshKey, infrastructure.detectPublicIp * property.useWithProperties * - * Admin token generation uses a direct ansible-playbook call inside - * rancher-go-test:latest (no make token target exists upstream yet). + * Admin token generation uses a direct ansible-playbook call inside the + * rancher-go-test image (no make token target exists upstream yet). */ def libraryBranch = env.QA_JENKINS_LIBRARY_BRANCH ?: 'main' @@ -42,6 +42,14 @@ library "qa-jenkins-library@${libraryBranch}" def DEFAULT_RANCHER_PASSWORD = 'X%S3bPyg@n8kWcRs675l#fH0' +// Build-scoped image tags: every direct docker invocation below runs the +// pinned tag so this job never depends on (or mutates) whatever some other +// job last built under the shared :latest tag on the same agent. The :latest +// alias is still applied at build time because qa-jenkins-library's +// make.runTarget resolves that name for its own container runs. +def INFRA_TOOLS_IMAGE = "rancher-infra-tools:pitupgrade-${env.BUILD_NUMBER}" +def GO_TEST_IMAGE = "rancher-go-test:pitupgrade-${env.BUILD_NUMBER}" + // Suite/method selectors passed as GOTEST_TESTCASE to the downstream executor. // Each phase is its own go test invocation (-run regex), so cross-phase state // lives in-cluster under deterministic names, never in process memory. @@ -116,7 +124,7 @@ def collectUpgradeDiagnostics() { -v ${workspace}:/workspace \ -e KUBECONFIG=/workspace/${env.INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml \ -w /workspace \ - rancher-infra-tools:latest \ + ${INFRA_TOOLS_IMAGE} \ sh -c ' set +e kubectl get nodes -o wide > diagnostics-nodes.txt 2>&1 @@ -387,10 +395,13 @@ rancherUpgradeInput: stage('Build Docker Images') { steps { - // rancher-infra-tools:latest — used by make.groovy / infra.* for all make targets - sh "docker build --no-cache --platform linux/amd64 -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." - // rancher-go-test:latest — used for qase binary build and admin-token playbook - sh "docker build --no-cache --platform linux/amd64 -t rancher-go-test:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.airgap-go-tests ." + // rancher-infra-tools — used by make.groovy / infra.* for all make + // targets (library resolves the :latest alias); direct invocations + // in this file use the build-pinned tag. + sh "docker build --no-cache --platform linux/amd64 -t ${INFRA_TOOLS_IMAGE} -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." + // rancher-go-test — used for the qase binary build, token playbook, + // and redaction step, always via the build-pinned tag. + sh "docker build --no-cache --platform linux/amd64 -t ${GO_TEST_IMAGE} -t rancher-go-test:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.airgap-go-tests ." } } @@ -693,7 +704,7 @@ rancher_use_bundled_system_charts: true -e ANSIBLE_CONFIG=/workspace/${env.INFRA_DIR}/ansible/ansible.cfg \ -v ${workspace}:/workspace \ -w /workspace/${env.INFRA_DIR}/ansible/rke2/default \ - rancher-go-test:latest \ + ${GO_TEST_IMAGE} \ ansible-playbook -i inventory/inventory.yml /workspace/${env.INFRA_DIR}/ansible/rancher/token/generate-admin-token.yml \ -e rancher_cattle_config_file=${containerConfigPath} \ -e rancher_token_ttl=${tokenTtl} \ @@ -710,7 +721,7 @@ rancher_use_bundled_system_charts: true // Redact secret fields from token JSON via Docker container. // Pipeline Utility Steps plugin (readJSON/writeJSON) is not installed, // and python3 is not available on the Jenkins agent host. - sh "docker run --rm -v \$(pwd):/workspace -w /workspace rancher-go-test:latest python3 -c \"import json; d=json.load(open('rancher-token.json')); [d.pop(k,None) for k in ('token','bearerToken','accessKey','secretKey','value')]; json.dump(d,open('rancher-token-metadata.json','w'),indent=2)\"" + sh "docker run --rm -v \$(pwd):/workspace -w /workspace ${GO_TEST_IMAGE} python3 -c \"import json; d=json.load(open('rancher-token.json')); [d.pop(k,None) for k in ('token','bearerToken','accessKey','secretKey','value')]; json.dump(d,open('rancher-token-metadata.json','w'),indent=2)\"" if (params.REPORT_ARTIFACTS) { archiveArtifacts artifacts: 'rancher-token-metadata.json' echo 'Archived redacted token metadata: rancher-token-metadata.json' @@ -750,7 +761,7 @@ rancher_use_bundled_system_charts: true docker run --rm --platform linux/amd64 \ -v ${workspace}:/workspace \ -w /workspace/${env.TESTS_DIR} \ - rancher-go-test:latest \ + ${GO_TEST_IMAGE} \ sh -c 'env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /workspace/testrun ./validation/pipeline/qase/testrun' """ @@ -971,7 +982,7 @@ rancher_use_bundled_system_charts: true } // Docker cleanup - sh 'docker rmi -f rancher-go-test:latest rancher-infra-tools:latest || true' + sh "docker rmi -f ${GO_TEST_IMAGE} ${INFRA_TOOLS_IMAGE} rancher-go-test:latest rancher-infra-tools:latest || true" // Sensitive artifact cleanup — shred or remove credentials from workspace sh ''' diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go index 379fb38b9e..bb4b77a54c 100644 --- a/validation/rancherupgrade/longhorn_test.go +++ b/validation/rancherupgrade/longhorn_test.go @@ -16,6 +16,7 @@ import ( actionsClusters "github.com/rancher/tests/actions/clusters" "github.com/rancher/tests/actions/projects" "github.com/rancher/tests/actions/storage" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) @@ -65,6 +66,7 @@ func (s *LonghornUpgradeTestSuite) TearDownSuite() { } func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { + logrus.Infof("pre-upgrade phase: creating project on cluster %s", s.cluster.ID) project, _, err := projects.CreateProjectAndNamespace(s.client, s.cluster.ID) require.NoError(s.T(), err) @@ -77,6 +79,7 @@ func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { require.NoError(s.T(), err) version, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: installing longhorn %s from the %s repo", version, catalog.RancherChartRepo) err = charts.InstallLonghornChart(s.client, charts.PayloadOpts{ Namespace: charts.LonghornNamespace, @@ -88,29 +91,36 @@ func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { }, }, nil) require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: longhorn %s installed; seeding checksum workload %s", version, checksumWorkloadName) err = storage.CreateChecksumWorkload(s.client, s.cluster.ID, charts.LonghornStorageClass, checksumWorkloadName, checksumContent) require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: checksum workload %s ready; pre-upgrade state seeded", checksumWorkloadName) } func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { cfg := loadUpgradeInput() require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") + logrus.Infof("post-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: server version %s observed; checking longhorn survival", cfg.TargetVersion) chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) require.NoError(s.T(), err) require.True(s.T(), chart.IsAlreadyInstalled, "longhorn chart must still be installed after the Rancher server upgrade") + logrus.Infof("post-upgrade phase: longhorn chart still installed; verifying workload data") err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: checksum verified on %s", checksumWorkloadName) } func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { cfg := loadUpgradeInput() require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") + logrus.Infof("chart-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) require.NoError(s.T(), err) @@ -122,6 +132,7 @@ func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { require.NoError(s.T(), err) latest, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) require.NoError(s.T(), err) + logrus.Infof("chart-upgrade phase: upgrading longhorn to %s from the upgraded catalog", latest) installOptions := &charts.InstallOptions{ Cluster: s.cluster, @@ -135,9 +146,11 @@ func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) require.NoError(s.T(), err) require.Equal(s.T(), latest, chart.ChartDetails.Spec.Chart.Metadata.Version) + logrus.Infof("chart-upgrade phase: longhorn at %s; verifying workload data", latest) err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) require.NoError(s.T(), err) + logrus.Infof("chart-upgrade phase: checksum verified on %s", checksumWorkloadName) } func TestLonghornUpgradeTestSuite(t *testing.T) { From b14003d896360936c8199a8d0c0c856612569ca4 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Mon, 19 Oct 2026 08:23:54 -0500 Subject: [PATCH 11/19] fix: decide teardown in post.always, defer artifact removal to cleanup Jenkins evaluates post conditions in a fixed order (always before failure, cleanup last), so the failure{} destroy block could never work: always had already shredded workspace_name.txt and tfvars and removed the infra-tools image teardown depends on, and with DESTROY_AFTER_TESTS=false + DESTROY_ON_FAILURE=true a failed run skipped teardown entirely, leaking the AWS environment. Fold the destroy decision into always (after diagnostics and Qase completion), gated on DESTROY_AFTER_TESTS or failure+DESTROY_ON_ FAILURE; move the docker rmi and sensitive-artifact shredding into a cleanup block so they run last. --- validation/Jenkinsfile.rancher-upgrade.e2e | 37 ++++++++++++---------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 90be3147cc..b53e126f2c 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -947,21 +947,13 @@ rancher_use_bundled_system_charts: true } post { - failure { - script { - if (params.DESTROY_ON_FAILURE) { - destroyInfrastructure('DESTROY_ON_FAILURE') - } - } - } - always { script { - // Failure diagnostics run here, not in the failure block: - // Jenkins evaluates post conditions in a fixed order with - // always first, so anything in failure{} would already have - // lost the docker images (rmi below) and the live cluster - // (destroy below). Collect while both still exist. + // Failure diagnostics run first, while the live cluster and + // the docker images still exist: Jenkins evaluates post + // conditions in a fixed order (always before failure, cleanup + // last), so every failure-path action must run in this block, + // ahead of the teardown below. if (currentBuild.currentResult != 'SUCCESS') { collectUpgradeDiagnostics() } @@ -977,11 +969,24 @@ rancher_use_bundled_system_charts: true } } - if (params.DESTROY_AFTER_TESTS) { - destroyInfrastructure('DESTROY_AFTER_TESTS') + // Single teardown decision point. The failure-path destroy is + // decided here rather than in a failure{} block because always + // runs first: destroyInfrastructure needs workspace_name.txt, + // the tfvars and the infra-tools image, which are only removed + // afterwards in cleanup{}. DESTROY_ON_FAILURE=false on a failed + // run keeps the environment around for debugging. + def destroyForFailure = currentBuild.currentResult != 'SUCCESS' && params.DESTROY_ON_FAILURE + if (params.DESTROY_AFTER_TESTS || destroyForFailure) { + destroyInfrastructure(params.DESTROY_AFTER_TESTS ? 'DESTROY_AFTER_TESTS' : 'DESTROY_ON_FAILURE') } - // Docker cleanup + } + } + + cleanup { + script { + // Docker cleanup — cleanup{} runs after always, so the + // teardown above can still resolve the infra-tools image. sh "docker rmi -f ${GO_TEST_IMAGE} ${INFRA_TOOLS_IMAGE} rancher-go-test:latest rancher-infra-tools:latest || true" // Sensitive artifact cleanup — shred or remove credentials from workspace From 84bac3d892463a06185882f846c08517db24dbda Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 20 Oct 2026 10:36:39 -0500 Subject: [PATCH 12/19] Address PR review: pit.weekly build tag, skip suite outside the upgrade pipeline --- validation/rancherupgrade/README.md | 6 +++++- validation/rancherupgrade/longhorn_test.go | 10 +++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md index 7436559c6b..6ef4686615 100644 --- a/validation/rancherupgrade/README.md +++ b/validation/rancherupgrade/README.md @@ -23,6 +23,10 @@ Do not "fix" the following; both are required by the upgrade workflow (rancher/t the cleanup. - The suite is admin-only: upgrade phases assert server-level state and the pipeline runs them with the admin token. +- The build tag is `(validation || pit.weekly)`: generic scheduled PIT legs deploy + Rancher once and never upgrade it, so `SetupSuite` skips the suite when + `rancherUpgradeInput.targetVersion` is unset; only the dedicated rancher-upgrade + pipeline sets it. ## Config @@ -38,7 +42,7 @@ rancherUpgradeInput: ``` `rancher.clusterName` selects the shared downstream cluster. `rancherUpgradeInput.targetVersion` -is required by the post-upgrade and chart-upgrade phases; the pipeline writes it from +is required by every phase (the suite skips without it); the pipeline writes it from the qa-infra-automation resolver output, or set it by hand for local runs. ## Running diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go index bb4b77a54c..e334d86f46 100644 --- a/validation/rancherupgrade/longhorn_test.go +++ b/validation/rancherupgrade/longhorn_test.go @@ -1,4 +1,4 @@ -//go:build validation && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 +//go:build (validation || pit.weekly) && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 package rancherupgrade @@ -47,6 +47,12 @@ func loadUpgradeInput() *UpgradeInput { } func (s *LonghornUpgradeTestSuite) SetupSuite() { + // Generic PIT legs never upgrade the server between phases; without an + // upgrade target the phases have nothing to assert, so skip the suite. + if loadUpgradeInput().TargetVersion == "" { + s.T().Skipf("%s.targetVersion is not set: this suite runs only in the rancher-upgrade pipeline, which upgrades the server between phases", UpgradeInputConfigKey) + } + testSession := session.NewSession() s.session = testSession @@ -100,7 +106,6 @@ func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { cfg := loadUpgradeInput() - require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") logrus.Infof("post-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) @@ -119,7 +124,6 @@ func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { cfg := loadUpgradeInput() - require.NotEmpty(s.T(), cfg.TargetVersion, "rancherUpgradeInput.targetVersion must be set for post-upgrade phases") logrus.Infof("chart-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) From 08487294e57028a1a58f963dd0d6bd3d6358577d Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Tue, 20 Oct 2026 10:46:07 -0500 Subject: [PATCH 13/19] Address PR review: fail on pipeline misconfig via rancherUpgradePipeline marker --- validation/Jenkinsfile.rancher-upgrade.e2e | 6 +++++- validation/rancherupgrade/README.md | 5 +++-- validation/rancherupgrade/longhorn_test.go | 13 ++++++++++--- validation/rancherupgrade/rancherupgrade.go | 11 +++++++++++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index b53e126f2c..29a01e7f39 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -330,7 +330,11 @@ rancher: # ("local") cluster; the upgrade suite installs Longhorn on it. clusterName: local rancherUpgradeInput: - targetVersion: "${RANCHER_VERSION_UPGRADE}"''', + targetVersion: "${RANCHER_VERSION_UPGRADE}" +# Marker: set only by this pipeline. The suite fails rather than skips when +# this marker is present but the target version is missing (pipeline bug). +rancherUpgradePipeline: + enabled: true''', ) booleanParam( name: 'REPORT_ARTIFACTS', diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md index 6ef4686615..0197f60c62 100644 --- a/validation/rancherupgrade/README.md +++ b/validation/rancherupgrade/README.md @@ -25,8 +25,9 @@ Do not "fix" the following; both are required by the upgrade workflow (rancher/t runs them with the admin token. - The build tag is `(validation || pit.weekly)`: generic scheduled PIT legs deploy Rancher once and never upgrade it, so `SetupSuite` skips the suite when - `rancherUpgradeInput.targetVersion` is unset; only the dedicated rancher-upgrade - pipeline sets it. + `rancherUpgradeInput.targetVersion` is unset. The dedicated pipeline marks its + config with `rancherUpgradePipeline.enabled: true`; with the marker present but + the target missing, the suite fails (pipeline bug) instead of skipping. ## Config diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go index e334d86f46..f1b5f5d7bb 100644 --- a/validation/rancherupgrade/longhorn_test.go +++ b/validation/rancherupgrade/longhorn_test.go @@ -47,9 +47,16 @@ func loadUpgradeInput() *UpgradeInput { } func (s *LonghornUpgradeTestSuite) SetupSuite() { - // Generic PIT legs never upgrade the server between phases; without an - // upgrade target the phases have nothing to assert, so skip the suite. - if loadUpgradeInput().TargetVersion == "" { + cfg := loadUpgradeInput() + pipeline := new(PipelineInput) + config.LoadConfig(PipelineConfigKey, pipeline) + + if cfg.TargetVersion == "" { + // The dedicated pipeline always writes both keys, so an empty target + // there is a pipeline bug and must fail, not skip. + require.False(s.T(), pipeline.Enabled, "rancher-upgrade pipeline config is missing %s.targetVersion", UpgradeInputConfigKey) + // Generic PIT legs never upgrade the server between phases; without + // an upgrade target the phases have nothing to assert, so skip. s.T().Skipf("%s.targetVersion is not set: this suite runs only in the rancher-upgrade pipeline, which upgrades the server between phases", UpgradeInputConfigKey) } diff --git a/validation/rancherupgrade/rancherupgrade.go b/validation/rancherupgrade/rancherupgrade.go index a08cf104eb..5d3aeb112a 100644 --- a/validation/rancherupgrade/rancherupgrade.go +++ b/validation/rancherupgrade/rancherupgrade.go @@ -8,6 +8,10 @@ package rancherupgrade const ( // UpgradeInputConfigKey is the cattle-config key holding upgrade-phase inputs. UpgradeInputConfigKey = "rancherUpgradeInput" + + // PipelineConfigKey marks a config as authored by the rancher-upgrade + // pipeline; generic PIT legs never set it. + PipelineConfigKey = "rancherUpgradePipeline" ) // UpgradeInput holds per-run upgrade parameters. TargetVersion is the Rancher @@ -16,3 +20,10 @@ const ( type UpgradeInput struct { TargetVersion string `json:"targetVersion" yaml:"targetVersion"` } + +// PipelineInput is the marker section the rancher-upgrade pipeline writes into +// its cattle-config; the suite fails rather than skips when it is present but +// the upgrade target is missing (pipeline bug, not a generic PIT leg). +type PipelineInput struct { + Enabled bool `json:"enabled" yaml:"enabled"` +} From da6002853f2e9f14ab3e1425ee87bc1bcc6afc30 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 21 Oct 2026 14:33:43 -0500 Subject: [PATCH 14/19] Address PR 986 review findings: checksum validation, stale catalog guard, flavor plumbing - actions/storage: reject checksum content outside [A-Za-z0-9-] before sh -c interpolation - rancherupgrade: fail fast when resolved latest equals the installed chart version (stale cluster-scoped catalog follower) - Jenkinsfile.rancher-upgrade.e2e: document timeout budget, plumb RANCHER_CHART_REPO_FLAVOR into make args, resolve kubeconfig via INFRA_DIR, select deploy repo by flavor - README + pit_schemas: correct downstream-cluster wording to the Rancher local cluster - actions/clusters: debug logging on server-version read/parse failures --- actions/clusters/local.go | 4 ++ actions/storage/checksum.go | 12 +++++ validation/Jenkinsfile.rancher-upgrade.e2e | 48 +++++++++++++------ validation/rancherupgrade/README.md | 29 ++++++----- validation/rancherupgrade/longhorn_test.go | 15 +++++- .../rancherupgrade/schemas/pit_schemas.yaml | 6 +-- 6 files changed, 82 insertions(+), 32 deletions(-) diff --git a/actions/clusters/local.go b/actions/clusters/local.go index 752df582ee..16940dfd53 100644 --- a/actions/clusters/local.go +++ b/actions/clusters/local.go @@ -56,11 +56,15 @@ func WaitRancherVersion(client *rancher.Client, targetVersion string, timeout ti serverVersionRawValue, err := client.Management.Setting.ByID(serverVersionSetting) if err != nil { // The server may be mid-restart during the upgrade rollout; retry the read. + // Logged so a terminal cause (e.g. a revoked token or unreachable API) + // is visible in the log instead of surfacing only as the final timeout. + logrus.Debugf("server-version read failed, retrying: %v", err) return false, nil } serverVersion, err := parseRancherVersion(serverVersionRawValue.Value) if err != nil { + logrus.Debugf("server-version value %q not parseable yet, retrying: %v", serverVersionRawValue.Value, err) return false, nil } diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go index 9f3d173cf7..6c5dcf2e59 100644 --- a/actions/storage/checksum.go +++ b/actions/storage/checksum.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "fmt" + "regexp" "strings" "github.com/rancher/shepherd/clients/rancher" @@ -26,6 +27,12 @@ import ( // ChecksumDataFile is the filename written under MountPath by CreateChecksumWorkload. const ChecksumDataFile = "checksum-data" +// checksumContentPattern constrains the content accepted by CreateChecksumWorkload: +// content is interpolated into a single-quoted sh -c command, so a value outside +// alphanumeric/hyphen (notably a quote) would break the quoting. Enforced in code +// so a future caller cannot silently inject shell. +var checksumContentPattern = regexp.MustCompile(`^[A-Za-z0-9-]+$`) + // CreateChecksumWorkload creates a 1GiB PVC named on storageClassName in the // default namespace, mounts it into a single-replica nginx deployment named at // MountPath, waits for rollout, then writes content to MountPath/ChecksumDataFile via @@ -34,6 +41,11 @@ const ChecksumDataFile = "checksum-data" // it is interpolated into an sh -c command. No session cleanup is registered: callers // are upgrade tests whose resources must outlive the process. func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageClassName string, name string, content string) error { + if !checksumContentPattern.MatchString(content) { + return fmt.Errorf("content %q must match %s: it is interpolated into a single-quoted sh -c command", + content, checksumContentPattern.String()) + } + storageClass, err := GetStorageClass(client, clusterID, storageClassName) if err != nil { return err diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e index 29a01e7f39..7b3b6ac24d 100755 --- a/validation/Jenkinsfile.rancher-upgrade.e2e +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -148,8 +148,11 @@ pipeline { options { ansiColor('xterm') - // Deploy + three sequential test phases + server upgrade + destroy - timeout(time: 240, unit: 'MINUTES') + // Budget: three sequential test phases (TEST_TIMEOUT each, default 60m) + // plus deploy, server upgrade, and destroy (~90m overhead). Must exceed + // the sum of the stage budgets or a legitimately slow run aborts + // mid-phase; revisit when raising TEST_TIMEOUT. + timeout(time: 300, unit: 'MINUTES') buildDiscarder(logRotator(numToKeepStr: '30')) } @@ -200,7 +203,7 @@ pipeline { string( name: 'RANCHER_CHART_REPO_FLAVOR', defaultValue: 'community', - description: 'Chart repo flavor for the upgrade (community|prime; prime is a later slice)' + description: 'Chart repo flavor (community|prime): selects the chart index for version resolution, the repo the pre-upgrade Rancher is deployed from, and the repo the server upgrade pulls from' ) // ── Infrastructure parameters ────────────────────────── @@ -232,7 +235,7 @@ pipeline { booleanParam( name: 'DESTROY_AFTER_TESTS', defaultValue: true, - description: 'Tear down infrastructure after tests complete' + description: 'Tear down infrastructure after a successful run; failed runs are governed by DESTROY_ON_FAILURE' ) // ── Terraform parameters ─────────────────────────────── @@ -288,7 +291,7 @@ nodes = [ defaultValue: '''--- kubernetes_version: ${RKE2_VERSION} cni: calico -kubeconfig_file: "/workspace/qa-infra-automation/ansible/rke2/default/kubeconfig.yaml" +kubeconfig_file: "/workspace/${INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml" cert_manager_version: "1.20.0" bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" password: "${RANCHER_ADMIN_PASSWORD}" @@ -449,7 +452,10 @@ rancherUpgradePipeline: def resolveOut = make.runTarget( target: 'rancher-resolve-version', dir: env.INFRA_DIR, - makeArgs: ['SOURCE_LINE=2.14'], + makeArgs: [ + 'SOURCE_LINE=2.14', + "RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}" + ], returnStdout: true, passAwsCreds: false ) @@ -472,6 +478,7 @@ rancherUpgradePipeline: def resolveOut = make.runTarget( target: 'rancher-resolve-version', dir: env.INFRA_DIR, + makeArgs: ["RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}"], returnStdout: true, passAwsCreds: false ) @@ -592,15 +599,19 @@ rancherUpgradePipeline: stage('Configure Ansible Variables') { sh "mkdir -p ${ansiblePath}/inventory/group_vars" - // The FROM version is a stable semver patch line, so - // its chart repo is the stable "latest" index. - def fromChartRepoUrl = 'https://releases.rancher.com/server-charts/latest' + // Flavor selects the deploy-side repo too: the prime + // leg installs the FROM version from the prime index + // (community stops the 2.14 line at 2.14.3 while prime + // continues it), keeping resolve, deploy, and upgrade + // all on the same artifact line. + def fromChartRepoUrl = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'https://charts.rancher.com/server-charts/prime' : 'https://releases.rancher.com/server-charts/latest' def upgradeRepoName = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'rancher-prime-upgrade' : 'rancher-upgrade' def ansibleVars = infrastructure.parseAndSubstituteVars( content: params.ANSIBLE_VARIABLES, envVars: [ 'RKE2_VERSION': params.RKE2_VERSION, + 'INFRA_DIR': env.INFRA_DIR, 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, 'RANCHER_VERSION': env.RANCHER_FROM_VERSION, @@ -977,11 +988,15 @@ rancher_use_bundled_system_charts: true // decided here rather than in a failure{} block because always // runs first: destroyInfrastructure needs workspace_name.txt, // the tfvars and the infra-tools image, which are only removed - // afterwards in cleanup{}. DESTROY_ON_FAILURE=false on a failed - // run keeps the environment around for debugging. - def destroyForFailure = currentBuild.currentResult != 'SUCCESS' && params.DESTROY_ON_FAILURE - if (params.DESTROY_AFTER_TESTS || destroyForFailure) { - destroyInfrastructure(params.DESTROY_AFTER_TESTS ? 'DESTROY_AFTER_TESTS' : 'DESTROY_ON_FAILURE') + // afterwards in cleanup{}. A failed run is governed by + // DESTROY_ON_FAILURE alone and a successful one by + // DESTROY_AFTER_TESTS, so DESTROY_ON_FAILURE=false on a failed + // run keeps the environment around for debugging even when + // DESTROY_AFTER_TESTS is true. + def buildSucceeded = currentBuild.currentResult == 'SUCCESS' + def shouldDestroy = buildSucceeded ? params.DESTROY_AFTER_TESTS : params.DESTROY_ON_FAILURE + if (shouldDestroy) { + destroyInfrastructure(buildSucceeded ? 'DESTROY_AFTER_TESTS' : 'DESTROY_ON_FAILURE') } } @@ -1028,9 +1043,12 @@ rancher_use_bundled_system_charts: true cleanup_file "workspace_name.txt" cleanup_file "resolve-from.json" cleanup_file "resolve-to.json" + # cattle-config carries the admin password and the live rotated + # admin token; group_vars/all.yml carries the rancher passwords. + cleanup_file "${TESTS_DIR}/cattle-config.yaml" if [ -n "${INFRA_DIR}" ] && [ -d "${INFRA_DIR}" ]; then - find "${INFRA_DIR}" -type f \\( -name 'terraform.tfvars' -o -name 'generated.tfvars' -o -name '*.pem' -o -name 'id_rsa' -o -name 'id_ed25519' \\) -exec sh -c ' + find "${INFRA_DIR}" -type f \\( -name 'terraform.tfvars' -o -name 'generated.tfvars' -o -name 'all.yml' -o -name '*.pem' -o -name 'id_rsa' -o -name 'id_ed25519' \\) -exec sh -c ' for file do if command -v shred >/dev/null 2>&1; then shred -u "$file" || rm -f "$file" diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md index 0197f60c62..c8161aea50 100644 --- a/validation/rancherupgrade/README.md +++ b/validation/rancherupgrade/README.md @@ -36,15 +36,17 @@ rancher: host: adminToken: insecure: true - clusterName: "" # pre-provisioned by qa-infra-automation + clusterName: "local" # Rancher's own local cluster: the qa-infra-automation RKE2 HA deploy rancherUpgradeInput: targetVersion: "" ``` - -`rancher.clusterName` selects the shared downstream cluster. `rancherUpgradeInput.targetVersion` -is required by every phase (the suite skips without it); the pipeline writes it from -the qa-infra-automation resolver output, or set it by hand for local runs. +`rancher.clusterName` selects the target cluster. The pipeline uses `local` — Rancher's own +3-node HA RKE2 cluster provisioned by qa-infra-automation — so Longhorn and its volume are +exercised on the server cluster itself, matching what the pipeline deploys. +`rancherUpgradeInput.targetVersion` is required by every phase (the suite skips without it); +the pipeline writes it from the qa-infra-automation resolver output, or set it by hand for +local runs. ## Running @@ -70,9 +72,9 @@ gotestsum --format standard-verbose --packages=github.com/rancher/tests/validati ## Notes -- The shared downstream RKE2 cluster is provisioned by qa-infra-automation and must be - sized for Longhorn plus the later NeuVector/Observability suites; this is an infra - concern and is not configurable here. +- The target cluster is Rancher's own local cluster (the qa-infra-automation 3-node HA + RKE2 deploy), so the server nodes themselves must be sized for Longhorn plus the later + NeuVector/Observability suites; this is an infra concern and is not configurable here. - The Rancher server upgrade itself is `make rancher-upgrade` in [qa-infra-automation](https://github.com/rancher/qa-infra-automation); this package only runs the pre- and post-upgrade assertions around it. @@ -80,8 +82,9 @@ gotestsum --format standard-verbose --packages=github.com/rancher/tests/validati `ansible/rancher/-ha/generated.tfvars`; the pipeline must copy it into the test config's `rancher.adminToken` before the post-upgrade phases (the pre-upgrade token is revoked by the upgrade). -- After a server upgrade the downstream cluster-scoped catalog follower can keep - serving the previous Rancher line's index (e.g. release-v2.14 after upgrading to - 2.15) until the downstream `cattle-cluster-agent` re-syncs. If the chart-upgrade - phase fails with "failed to find chart longhorn from the chart repo", restart the - downstream cluster-agent and retry; observed re-sync time ~4 minutes. +- After a server upgrade the cluster-scoped catalog follower can keep serving the + previous Rancher line's index (e.g. release-v2.14 after upgrading to 2.15) until the + `cattle-cluster-agent` re-syncs. The chart-upgrade phase detects this and fails with + an explicit stale-catalog message when the catalog's latest equals the installed + version, or with "failed to find chart longhorn from the chart repo" on a lookup + miss; restart the cluster agent and re-run the phase. Observed re-sync time ~4 minutes. diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go index f1b5f5d7bb..3b203420d6 100644 --- a/validation/rancherupgrade/longhorn_test.go +++ b/validation/rancherupgrade/longhorn_test.go @@ -143,7 +143,20 @@ func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { require.NoError(s.T(), err) latest, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) require.NoError(s.T(), err) - logrus.Infof("chart-upgrade phase: upgrading longhorn to %s from the upgraded catalog", latest) + + // Guard against a stale cluster-scoped catalog: until the cluster agent + // re-syncs after the server upgrade, the follower can still serve the + // previous Rancher line's index, whose latest equals the installed version. + // Upgrading to it would pass vacuously (a same-version upgrade asserted + // equal to itself), so fail with the runbook remedy instead; see README Notes. + installed, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.NotEqual(s.T(), + installed.ChartDetails.Spec.Chart.Metadata.Version, latest, + "cluster-scoped catalog still serves the pre-upgrade index: installed longhorn %s equals latest %s; restart the cluster agent and re-run this phase", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + logrus.Infof("chart-upgrade phase: upgrading longhorn %s to %s from the upgraded catalog", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) installOptions := &charts.InstallOptions{ Cluster: s.cluster, diff --git a/validation/rancherupgrade/schemas/pit_schemas.yaml b/validation/rancherupgrade/schemas/pit_schemas.yaml index da764c080d..a95dc71f59 100644 --- a/validation/rancherupgrade/schemas/pit_schemas.yaml +++ b/validation/rancherupgrade/schemas/pit_schemas.yaml @@ -5,7 +5,7 @@ description: "Install Longhorn and seed a checksum workload on a Longhorn volume before the Rancher server upgrade" automation: 2 steps: - - action: "Create a project on the downstream cluster" + - action: "Create a project on the Rancher local cluster" data: "" expectedresult: "The project is created successfully" position: 1 @@ -47,11 +47,11 @@ position: 1 - action: "Resolve the latest Longhorn chart version from the cluster-scoped catalog" data: "Chart: longhorn from the rancher-charts repo" - expectedresult: "The latest cluster-scoped chart version is resolved" + expectedresult: "The latest cluster-scoped chart version is resolved and differs from the installed pre-upgrade version" position: 2 - action: "Upgrade the Longhorn chart with retry" data: "longhorn and longhorn-crd to the latest version" - expectedresult: "Both charts reach Deployed state and workloads become Ready" + expectedresult: "The longhorn chart reaches Deployed state and workloads become Ready" position: 3 - action: "Verify the chart version and the checksum workload" data: "Chart version equals latest; PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" From a19485d98e37dab2dd3494359fb6faf48802cc9d Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 21 Oct 2026 14:38:49 -0500 Subject: [PATCH 15/19] Address hamistao review findings: dedupe deployment helper, clarify version scope - pvc.go: createNginxDeploymentWithPVC takes an explicit deploymentName, so CreatePVCWorkload keeps random names while CreateChecksumWorkload passes its deterministic one; shared construction replaces ~25 duplicated lines - checksum.go: reuse the shared helper; drop now-unused charts/wloads imports - README: previous-release-line-to-latest wording (pipeline default, not a suite constraint) and a deliberate-deviation bullet for the out-of-band server upgrade --- actions/storage/checksum.go | 38 ++++------------------------- actions/storage/pvc.go | 23 ++++++++++------- validation/rancherupgrade/README.md | 15 ++++++++++-- 3 files changed, 32 insertions(+), 44 deletions(-) diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go index 6c5dcf2e59..e7da7d5b29 100644 --- a/actions/storage/checksum.go +++ b/actions/storage/checksum.go @@ -9,13 +9,11 @@ import ( "github.com/rancher/shepherd/clients/rancher" steveV1 "github.com/rancher/shepherd/clients/rancher/v1" - "github.com/rancher/shepherd/extensions/charts" "github.com/rancher/shepherd/extensions/defaults" "github.com/rancher/shepherd/extensions/defaults/namespaces" "github.com/rancher/shepherd/extensions/defaults/stevetypes" "github.com/rancher/shepherd/extensions/kubeapi/cluster" "github.com/rancher/shepherd/extensions/kubeconfig" - wloads "github.com/rancher/shepherd/extensions/workloads" "github.com/rancher/tests/actions/kubeapi/volumes/persistentvolumeclaims" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" @@ -94,37 +92,11 @@ func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageCla } logrus.Debugf("PVC %s is Bound to volume %s", pvc.Name, pvc.Spec.VolumeName) - volMount := corev1.VolumeMount{ - MountPath: MountPath, - Name: name, - } - - podVol := corev1.Volume{ - Name: name, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: name, - }, - }, - } - - steveclient, err := client.Steve.ProxyDownstream(clusterID) - if err != nil { - return err - } - - containerTemplate := wloads.NewContainer(nginxName, nginxName, corev1.PullAlways, []corev1.VolumeMount{volMount}, []corev1.EnvFromSource{}, nil, nil, nil) - podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: name}, nil) - deployment := wloads.NewDeploymentTemplate(name, namespaces.Default, podTemplate, true, nil) - - _, err = steveclient.SteveType(stevetypes.Deployment).Create(deployment) - if err != nil { - return err - } - - err = charts.WatchAndWaitDeployments(client, clusterID, namespaces.Default, metav1.ListOptions{ - FieldSelector: "metadata.name=" + name, - }) + // Shared nginx+PVC deployment construction with CreatePVCWorkload; the + // deterministic name makes PVC, volume mount, deployment, and relocation + // label all equal name, so later phases re-find the workload from a + // separate process. + _, err = createNginxDeploymentWithPVC(client, clusterID, name, name, name) if err != nil { return err } diff --git a/actions/storage/pvc.go b/actions/storage/pvc.go index 163c0df025..4030e6823d 100644 --- a/actions/storage/pvc.go +++ b/actions/storage/pvc.go @@ -158,7 +158,6 @@ func CreatePVC(client *rancher.Client, clusterID string, storageClassName string } return false, err }) - if err != nil { return nil, "", err } @@ -185,19 +184,25 @@ func CreatePVCWorkload(t *testing.T, client *rancher.Client, clusterID string, s persistentVolumeClaim, volumeName, err := CreatePVC(client, clusterID, storageClassName) require.NoError(t, err) - logrus.Debugf("Creating Nginx deployment with %s PVC (volume name %s)", persistentVolumeClaim.Name, volumeName) - nginxResponse, err := createNginxDeploymentWithPVC(client, clusterID, persistentVolumeClaim.Name, volumeName) + // Random workload name: this helper targets single-process functional tests; + // callers needing a deterministic (cross-phase relocatable) name call + // createNginxDeploymentWithPVC directly (see CreateChecksumWorkload). + deploymentName := namegenerator.AppendRandomString(containerNamePrefix) + logrus.Debugf("Creating Nginx deployment %s with %s PVC (volume name %s)", deploymentName, persistentVolumeClaim.Name, volumeName) + nginxResponse, err := createNginxDeploymentWithPVC(client, clusterID, persistentVolumeClaim.Name, volumeName, deploymentName) require.NoError(t, err) return nginxResponse } -// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace -func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcName, volName string) (*steveV1.SteveAPIObject, error) { +// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace. +// The deployment and its DeploymentIdentifierLabel both use deploymentName, letting callers +// pick a random name (functional tests) or a deterministic name (workloads a later test +// phase must re-find from a separate process). +func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcName, volName, deploymentName string) (*steveV1.SteveAPIObject, error) { logrus.Tracef("Vol: %s", volName) logrus.Tracef("Pod: %s", pvcName) - containerName := namegenerator.AppendRandomString(containerNamePrefix) volMount := &corev1.VolumeMount{ MountPath: MountPath, Name: volName, @@ -218,8 +223,8 @@ func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcN } containerTemplate := wloads.NewContainer(nginxName, nginxName, corev1.PullAlways, []corev1.VolumeMount{*volMount}, []corev1.EnvFromSource{}, nil, nil, nil) - podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: containerName}, nil) - deployment := wloads.NewDeploymentTemplate(containerName, namespaces.Default, podTemplate, true, nil) + podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: deploymentName}, nil) + deployment := wloads.NewDeploymentTemplate(deploymentName, namespaces.Default, podTemplate, true, nil) deploymentResp, err := steveclient.SteveType(stevetypes.Deployment).Create(deployment) if err != nil { @@ -227,7 +232,7 @@ func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcN } err = charts.WatchAndWaitDeployments(client, clusterID, namespaces.Default, metav1.ListOptions{ - FieldSelector: "metadata.name=" + containerName, + FieldSelector: "metadata.name=" + deploymentName, }) return deploymentResp, err diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md index c8161aea50..901b0f5b32 100644 --- a/validation/rancherupgrade/README.md +++ b/validation/rancherupgrade/README.md @@ -1,7 +1,12 @@ # Rancher Upgrade -The `rancherupgrade` package validates upgrading the Rancher server (2.14.x -> latest) -with interoperability products installed. The first vertical is Longhorn: a pre-upgrade +The `rancherupgrade` package validates upgrading the Rancher server from the previous +release line to the latest (currently 2.14.x -> 2.15.x) with interoperability products +installed. The line pair is the pipeline's default, not a suite constraint: the suite +itself is version-agnostic and gates on `rancherUpgradeInput.targetVersion`, while the +pipeline resolves the source via `RANCHER_VERSION_FROM` (empty = latest 2.14.x patch via +`SOURCE_LINE=2.14`) and the target via `RANCHER_VERSION_TO_UPGRADE` (empty = latest +released final). The first vertical is Longhorn: a pre-upgrade phase installs the Longhorn chart and seeds a checksum workload on a Longhorn volume, the Rancher server is then upgraded out-of-band (qa-infra-automation `make rancher-upgrade`), and post-upgrade phases verify the server version, the surviving Longhorn installation, @@ -23,6 +28,12 @@ Do not "fix" the following; both are required by the upgrade workflow (rancher/t the cleanup. - The suite is admin-only: upgrade phases assert server-level state and the pipeline runs them with the admin token. +- The server upgrade stays out-of-band (a pipeline stage running qa-infra-automation + `make rancher-upgrade`) rather than a step inside the test: the go test phases + assert outcomes only and hold no infra credentials, the upgrade needs the + controller-side qa-infra checkout and its terraform/ansible credentials, and + per-phase Jenkins stage status is the acceptance signal; pulling the upgrade into + the test process would collapse the three stages into one. - The build tag is `(validation || pit.weekly)`: generic scheduled PIT legs deploy Rancher once and never upgrade it, so `SetupSuite` skips the suite when `rancherUpgradeInput.targetVersion` is unset. The dedicated pipeline marks its From 0be13ac59abecab76a030128432f52e95baa9c29 Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 22 Oct 2026 08:31:43 -0500 Subject: [PATCH 16/19] Add NeuVector suite to rancherupgrade package (#969) Add the NeuVector vertical to the Rancher server upgrade validation: - actions/neuvector/admission.go: reusable, error-returning admission enforcement helpers. EnableAdmissionControlDenyNamespace applies the NvAdmissionControlSecurityRule CR (documented name "local") in protect mode with a namespace-criterion deny rule (containsAny op), creates the deny namespace and waits for the neuvector-validating-admission-webhook ValidatingWebhookConfiguration. VerifyAdmissionDenial attempts a known-bad pod create and proves the API server rejects it with an admission-webhook denial, tolerating controller rule reloads within a bounded window. CRD shape grounded in the neuvector-crd chart schema and the NeuVector console-exported manifest docs. - actions/charts: UpgradeNeuVectorChart upgrades neuvector + neuvector-crd with stale-catalog-safe action retry, PendingUpgrade->Deployed watches and workload readiness, carrying over the installed release values (helm --reuse-values semantics) so registry, hardening and k3s settings survive instead of a blind version bump. - validation/charts: export VerifyNeuVectorServicesActive and VerifyNeuVectorWebUIAccessible for cross-package reuse (move-not-copy; in-package callers updated). - validation/rancherupgrade: NeuVectorUpgradeTestSuite with the three -run-selectable phases (pre-upgrade install + denial proof, post-upgrade server-version gate + services/web UI/enforcement survival, chart upgrade to the cluster-scoped latest + health/enforcement re-verification), following the Longhorn suite contract. - schemas: PIT schema entries for the three new test methods. --- actions/charts/neuvector.go | 136 +++++++++ actions/neuvector/admission.go | 278 ++++++++++++++++++ validation/charts/neuvector.go | 8 +- validation/charts/neuvector_test.go | 4 +- validation/rancherupgrade/neuvector_test.go | 235 +++++++++++++++ .../rancherupgrade/schemas/pit_schemas.yaml | 70 +++++ 6 files changed, 725 insertions(+), 6 deletions(-) create mode 100644 actions/neuvector/admission.go create mode 100644 validation/rancherupgrade/neuvector_test.go diff --git a/actions/charts/neuvector.go b/actions/charts/neuvector.go index 28786b1563..b94c0f0321 100644 --- a/actions/charts/neuvector.go +++ b/actions/charts/neuvector.go @@ -5,13 +5,18 @@ import ( "encoding/json" "time" + catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" "github.com/rancher/shepherd/clients/rancher" "github.com/rancher/shepherd/clients/rancher/catalog" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/defaults" "github.com/rancher/shepherd/pkg/api/steve/catalog/types" + "github.com/rancher/shepherd/pkg/wait" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kwait "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" ) const ( @@ -178,6 +183,137 @@ func InstallNeuVectorChart(client *rancher.Client, payload PayloadOpts) error { return nil } +// UpgradeNeuVectorChart upgrades the neuvector and neuvector-crd charts to +// installOptions.Version on installOptions.Cluster, waits for the Deployed state +// and for the NeuVector workloads (controller deployment, enforcer daemonset) to +// become Ready again. +// +// The upgrade carries over the values of the installed release (the App resource's +// spec.values, equivalent to helm --reuse-values) instead of applying a blind +// version bump: registry overrides required by airgap environments and the +// enforcer hardening or k3s options set at install time must survive the upgrade, +// or the upgraded NeuVector would come back with chart defaults (docker.io +// images, unhardened enforcer) and break those environments. The neuvector-crd +// chart takes no values, mirroring the install split. +// +// No session cleanup is registered: the uninstall cleanup belongs to the install +// call of the same session, and upgrade-test sessions must not delete the chart. +func UpgradeNeuVectorChart(client *rancher.Client, installOptions *InstallOptions) error { + serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) + if err != nil { + return err + } + + registrySetting, err := client.Management.Setting.ByID(defaultRegistrySettingID) + if err != nil { + return err + } + + catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + // Reading the installed App yields exactly the values overridden at install + // time (chart defaults are not stored), which is what must be re-applied. + installedApp, err := catalogClient.Apps(NeuVectorNamespace).Get(context.TODO(), NeuVectorChartName, metav1.GetOptions{}) + if err != nil { + return err + } + + neuVectorChartUpgradeActionPayload := &PayloadOpts{ + InstallOptions: *installOptions, + Name: NeuVectorChartName, + Namespace: NeuVectorNamespace, + Host: serverSetting.Value, + DefaultRegistry: registrySetting.Value, + } + + chartUpgradeAction := newNeuVectorChartUpgradeAction(neuVectorChartUpgradeActionPayload, map[string]interface{}(installedApp.Spec.Values)) + + bodyBytes, err := json.Marshal(chartUpgradeAction) + if err != nil { + return err + } + err = ChartActionWithRetry(context.TODO(), client, verbUpgrade, neuVectorChartUpgradeActionPayload, catalog.RancherChartRepo, []string{NeuVectorChartName, NeuVectorChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbUpgrade, bodyBytes)) + if err != nil { + return err + } + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + if err != nil { + return err + } + + adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + watchAppInterface, err := adminCatalogClient.Apps(NeuVectorNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + NeuVectorChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusPendingUpgrade) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + watchAppInterface, err = adminCatalogClient.Apps(NeuVectorNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + NeuVectorChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusDeployed) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, NeuVectorNamespace, metav1.ListOptions{}) + if err != nil { + return err + } + + return shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, NeuVectorNamespace, metav1.ListOptions{}) +} + +// newNeuVectorChartUpgradeAction is a private helper function that returns the +// chart upgrade action for NeuVector, reusing the installed release values on the +// main chart. OpenAPI validation stays enabled: the neuvector CRDs are small and +// do not hit the schema-conflict failures seen on large CRD charts. +func newNeuVectorChartUpgradeAction(p *PayloadOpts, installedValues map[string]interface{}) *types.ChartUpgradeAction { + chartUpgrade := NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, installedValues) + chartUpgradeCRD := NewChartUpgrade(p.Name+"-crd", p.Name+"-crd", p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, nil) + chartUpgrades := []types.ChartUpgrade{*chartUpgradeCRD, *chartUpgrade} + + return NewChartUpgradeAction(p.Namespace, chartUpgrades) +} + func uninstallNeuVectorChart(client *rancher.Client, namespace string, clusterID string) error { catalogClient, err := client.GetClusterCatalogClient(clusterID) if err != nil { diff --git a/actions/neuvector/admission.go b/actions/neuvector/admission.go new file mode 100644 index 0000000000..58eda115d4 --- /dev/null +++ b/actions/neuvector/admission.go @@ -0,0 +1,278 @@ +// Package neuvector provides reusable helpers around NeuVector on Rancher-managed +// clusters: test configuration and admission-control enforcement exercised by the +// validation suites. +package neuvector + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/rancher/shepherd/clients/rancher" + v1 "github.com/rancher/shepherd/clients/rancher/v1" + "github.com/rancher/shepherd/extensions/defaults/stevetypes" + "github.com/rancher/tests/actions/namespaces" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kwait "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + // AdmissionControlSteveType is the steve type of the cluster-scoped + // NvAdmissionControlSecurityRule custom resource served by the + // neuvector-crd chart (group neuvector.com, singular nvadmissioncontrolsecurityrule). + AdmissionControlSteveType = "neuvector.com.nvadmissioncontrolsecurityrule" + + // AdmissionControlCRName is the resource name the NeuVector CRD documentation + // requires for the admission control custom resource: always "local". + AdmissionControlCRName = "local" + + // ValidatingWebhookSteveType is the steve type of the Kubernetes + // ValidatingWebhookConfiguration resource on the downstream cluster. + ValidatingWebhookSteveType = "admissionregistration.k8s.io.validatingwebhookconfiguration" + + // NeuVectorValidatingWebhookName is the ValidatingWebhookConfiguration NeuVector + // registers automatically once admission control is enabled. + NeuVectorValidatingWebhookName = "neuvector-validating-admission-webhook" + + // admissionWebhookDenialMarker is the substring a Kubernetes API server error + // contains when a create is rejected by an admission webhook; matched + // case-insensitively. + admissionWebhookDenialMarker = "admission webhook" + + admissionPollInterval = 5 * time.Second + admissionPollTimeout = 3 * time.Minute + + // admissionProbePodName is the deterministic name of the pod whose creation + // the admission deny rule must reject; it never starts running, so the name + // stays free for every verification attempt. + admissionProbePodName = "rancher-admission-deny-probe" + + // admissionProbePodImage is only referenced by the denied create request and + // is never pulled while the webhook enforces the deny rule. + admissionProbePodImage = "busybox:1.36" + + admissionProbeContainerName = "deny-probe" +) + +// EnableAdmissionControlDenyNamespace enables NeuVector admission control on the +// cluster and applies a protect-mode deny rule rejecting any workload create in +// denyNamespace. The NvAdmissionControlSecurityRule custom resource is created +// (or updated) under the NeuVector-documented name "local" with the shape of the +// console-exported manifest: config enable/protect/service plus a deny rule whose +// namespace criterion uses the containsAny operator. The deny namespace is created +// if missing, and the call returns only after NeuVector has registered its +// validating webhook, so enforcement can be proven right away with +// VerifyAdmissionDenial. Everything is named deterministically and nothing is +// registered for session cleanup: upgrade-test phases must keep the rule in force +// across separate go test processes. +func EnableAdmissionControlDenyNamespace(client *rancher.Client, clusterID, denyNamespace string) error { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return err + } + + if err := ensureNamespace(steveClient, denyNamespace); err != nil { + return err + } + + if err := applyAdmissionControlRule(steveClient, denyNamespace); err != nil { + return err + } + + return waitForAdmissionWebhook(steveClient) +} + +// VerifyAdmissionDenial proves the NeuVector admission webhook enforces the deny +// rule for denyNamespace: it attempts to create a pod there and expects the API +// server to reject the create with an admission-webhook denial. Attempts are +// retried within a bounded window because the NeuVector controllers reload the +// CRD rules asynchronously (for example while rolling after a chart upgrade), and +// a pod created before the rule is loaded again is deleted before the next +// attempt. An unexpected create success that outlives the window, or a persistent +// non-webhook error, is returned as an error. +func VerifyAdmissionDenial(client *rancher.Client, clusterID, denyNamespace string) error { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return err + } + + var lastErr error + err = kwait.PollUntilContextTimeout(context.Background(), admissionPollInterval, admissionPollTimeout, true, func(ctx context.Context) (bool, error) { + lastErr = attemptAdmissionProbeCreate(steveClient, denyNamespace) + switch { + case lastErr == nil: + // The deny rule did not fire: NeuVector may still be loading the CRD + // rules after a restart, so remove the stray pod and retry. + logrus.Warnf("admission probe pod in namespace %s was created instead of denied; deleting it and retrying", denyNamespace) + if deleteErr := deleteAdmissionProbePod(steveClient, denyNamespace); deleteErr != nil { + return false, deleteErr + } + lastErr = fmt.Errorf("pod %s/%s was created but the NeuVector admission webhook must deny workload creates in this namespace", denyNamespace, admissionProbePodName) + return false, nil + case isAdmissionWebhookDenial(lastErr): + logrus.Infof("NeuVector admission webhook denied the probe create in namespace %s as expected", denyNamespace) + return true, nil + default: + // Retry transient API errors, but keep the surprise for the timeout. + logrus.Warnf("probe create in namespace %s failed with a non-webhook error, retrying: %v", denyNamespace, lastErr) + return false, nil + } + }) + if err != nil { + return fmt.Errorf("NeuVector admission enforcement not proven for namespace %s: %w", denyNamespace, lastErr) + } + + return nil +} + +// applyAdmissionControlRule creates or updates the NvAdmissionControlSecurityRule +// named "local" with the deny rule for denyNamespace. +func applyAdmissionControlRule(steveClient *v1.Client, denyNamespace string) error { + crClient := steveClient.SteveType(AdmissionControlSteveType) + + existing, err := crClient.ByID(AdmissionControlCRName) + if err != nil { + if !k8sErrors.IsNotFound(err) { + return err + } + + cr := &v1.SteveAPIObject{} + cr.TypeMeta = metav1.TypeMeta{Kind: "NvAdmissionControlSecurityRule", APIVersion: "neuvector.com/v1"} + cr.Name = AdmissionControlCRName + cr.Spec = admissionControlSpec(denyNamespace) + + logrus.Infof("creating NeuVector admission control rule denying workload creates in namespace %s", denyNamespace) + _, err = crClient.Create(cr) + return err + } + + updated := *existing + updated.Spec = admissionControlSpec(denyNamespace) + + logrus.Infof("updating NeuVector admission control rule denying workload creates in namespace %s", denyNamespace) + _, err = crClient.Update(existing, &updated) + return err +} + +// admissionControlSpec builds the spec of the admission control custom resource +// following the console-exported manifest documented by NeuVector: admission +// control enabled in protect mode through the service client, plus a deny rule +// matching the deny namespace with the containsAny operator on the namespace +// criterion. Protect mode is what turns the deny action into an inline rejection. +func admissionControlSpec(denyNamespace string) map[string]any { + return map[string]any{ + "config": map[string]any{ + "enable": true, + "mode": "protect", + "client_mode": "service", + }, + "rules": []any{ + map[string]any{ + "action": "deny", + "containers": []any{"containers"}, + "criteria": []any{ + map[string]any{ + "name": "namespace", + "op": "containsAny", + "path": "namespace", + "value": denyNamespace, + }, + }, + "disabled": false, + "rule_mode": "", + }, + }, + } +} + +// waitForAdmissionWebhook polls for the ValidatingWebhookConfiguration NeuVector +// registers once admission control is enabled; until it exists the API server +// cannot consult NeuVector at all. +func waitForAdmissionWebhook(steveClient *v1.Client) error { + webhookClient := steveClient.SteveType(ValidatingWebhookSteveType) + + return kwait.PollUntilContextTimeout(context.Background(), admissionPollInterval, admissionPollTimeout, true, func(ctx context.Context) (bool, error) { + _, err := webhookClient.ByID(NeuVectorValidatingWebhookName) + if err != nil { + if k8sErrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + return true, nil + }) +} + +// ensureNamespace creates the namespace if it does not exist yet; a namespace +// create is not a workload create, so the admission deny rule does not reject it. +func ensureNamespace(steveClient *v1.Client, name string) error { + nsClient := steveClient.SteveType(namespaces.NamespaceSteveType) + + _, err := nsClient.ByID(name) + if err == nil { + return nil + } + if !k8sErrors.IsNotFound(err) { + return err + } + + ns := &v1.SteveAPIObject{} + ns.TypeMeta = metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"} + ns.Name = name + + logrus.Infof("creating namespace %s for the admission deny probe", name) + if _, err = nsClient.Create(ns); err != nil { + // Tolerate a create race by re-checking existence. + if _, getErr := nsClient.ByID(name); getErr != nil { + return err + } + } + + return nil +} + +// attemptAdmissionProbeCreate tries to create the probe pod whose name and image +// are fixed; a nil return means the create was accepted. +func attemptAdmissionProbeCreate(steveClient *v1.Client, namespace string) error { + probe := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: admissionProbePodName, + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: admissionProbeContainerName, + Image: admissionProbePodImage, + }, + }, + }, + } + + _, err := steveClient.SteveType(stevetypes.Pod).Create(probe) + return err +} + +// deleteAdmissionProbePod removes a probe pod that escaped the deny rule. +func deleteAdmissionProbePod(steveClient *v1.Client, namespace string) error { + probe, err := steveClient.SteveType(stevetypes.Pod).ByID(namespace + "/" + admissionProbePodName) + if err != nil { + if k8sErrors.IsNotFound(err) { + return nil + } + return err + } + + return steveClient.SteveType(stevetypes.Pod).Delete(probe) +} + +// isAdmissionWebhookDenial reports whether err is a Kubernetes admission-webhook +// rejection, the error shape the API server returns for a denied create. +func isAdmissionWebhookDenial(err error) bool { + return err != nil && strings.Contains(strings.ToLower(err.Error()), admissionWebhookDenialMarker) +} diff --git a/validation/charts/neuvector.go b/validation/charts/neuvector.go index 0a16019bf8..47027ed55e 100644 --- a/validation/charts/neuvector.go +++ b/validation/charts/neuvector.go @@ -26,9 +26,9 @@ var expectedNeuVectorServices = []string{ neuVectorAdmissionWebhookSvc, } -// verifyNeuVectorServicesActive checks that all expected NeuVector services exist and are active +// VerifyNeuVectorServicesActive checks that all expected NeuVector services exist and are active // in the NeuVector namespace. -func verifyNeuVectorServicesActive(client *rancher.Client, clusterID string) error { +func VerifyNeuVectorServicesActive(client *rancher.Client, clusterID string) error { steveClient, err := client.Steve.ProxyDownstream(clusterID) if err != nil { return err @@ -51,9 +51,9 @@ func verifyNeuVectorServicesActive(client *rancher.Client, clusterID string) err return nil } -// verifyNeuVectorWebUIAccessible verifies that the NeuVector manager web UI is reachable +// VerifyNeuVectorWebUIAccessible verifies that the NeuVector manager web UI is reachable // through the Rancher service proxy endpoint. -func verifyNeuVectorWebUIAccessible(client *rancher.Client, clusterID string) (string, error) { +func VerifyNeuVectorWebUIAccessible(client *rancher.Client, clusterID string) (string, error) { proxyPath := fmt.Sprintf(neuVectorWebUIServiceProxyFmt, clusterID, charts.NeuVectorNamespace, neuVectorWebUIService, neuVectorWebUIPort) logrus.Infof("Verifying NeuVector manager UI is accessible at path [%s]", proxyPath) diff --git a/validation/charts/neuvector_test.go b/validation/charts/neuvector_test.go index 140577a612..5822ac87e9 100644 --- a/validation/charts/neuvector_test.go +++ b/validation/charts/neuvector_test.go @@ -108,7 +108,7 @@ func (n *NeuVectorTestSuite) TestNeuVectorInstallation() { require.NoError(n.T(), err) n.T().Log("Verifying all expected NeuVector services are active") - err = verifyNeuVectorServicesActive(client, n.cluster.ID) + err = VerifyNeuVectorServicesActive(client, n.cluster.ID) require.NoError(n.T(), err) if n.chartInstallOptions.DefaultRegistry != "" { @@ -119,7 +119,7 @@ func (n *NeuVectorTestSuite) TestNeuVectorInstallation() { } n.T().Log("Verifying NeuVector manager web UI is accessible via service proxy") - respUI, err := verifyNeuVectorWebUIAccessible(client, n.cluster.ID) + respUI, err := VerifyNeuVectorWebUIAccessible(client, n.cluster.ID) require.NoError(n.T(), err, "NeuVector manager web UI should be accessible via Rancher service proxy") require.NotEmpty(n.T(), respUI, "Expected non-empty response from NeuVector manager web UI") } diff --git a/validation/rancherupgrade/neuvector_test.go b/validation/rancherupgrade/neuvector_test.go new file mode 100644 index 0000000000..bea6d59931 --- /dev/null +++ b/validation/rancherupgrade/neuvector_test.go @@ -0,0 +1,235 @@ +//go:build (validation || pit.weekly) && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 + +package rancherupgrade + +import ( + "fmt" + "testing" + + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + management "github.com/rancher/shepherd/clients/rancher/generated/management/v3" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + "github.com/rancher/tests/actions/charts" + actionsClusters "github.com/rancher/tests/actions/clusters" + "github.com/rancher/tests/actions/neuvector" + "github.com/rancher/tests/actions/projects" + validationCharts "github.com/rancher/tests/validation/charts" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type NeuVectorUpgradeTestSuite struct { + suite.Suite + session *session.Session + client *rancher.Client + cluster *clusters.ClusterMeta +} + +// nvAdmissionDenyNamespace is the deterministic namespace protected by the +// NeuVector admission control deny rule: workload creates in it must be +// rejected by the admission webhook before and after the Rancher server +// upgrade and again after the NeuVector chart upgrade. The deny namespace and +// the NvAdmissionControlSecurityRule are created in the pre-upgrade phase and +// must survive every later phase, which run as separate go test processes. +const nvAdmissionDenyNamespace = "rancher-upgrade-neuvector-deny" + +// nvSystemProject returns the System project of the target cluster, which the +// NeuVector chart is installed into, the same placement the hardened NeuVector +// suite uses. +func nvSystemProject(client *rancher.Client, clusterID string) (*management.Project, error) { + project, err := projects.GetProjectByName(client, clusterID, charts.SystemProject) + if err != nil { + return nil, err + } + + if project.Name != charts.SystemProject { + return nil, fmt.Errorf("expected project %q on cluster %s, got %q", charts.SystemProject, clusterID, project.Name) + } + + return project, nil +} + +// nvSystemDefaultRegistry returns the system-default-registry setting value. It +// is empty outside airgap environments and must be passed to the NeuVector chart +// install so the images come from the mirrored registry in airgap environments. +func nvSystemDefaultRegistry(client *rancher.Client) (string, error) { + registrySetting, err := client.Management.Setting.ByID("system-default-registry") + if err != nil { + return "", err + } + + return registrySetting.Value, nil +} + +func (s *NeuVectorUpgradeTestSuite) SetupSuite() { + cfg := loadUpgradeInput() + pipeline := new(PipelineInput) + config.LoadConfig(PipelineConfigKey, pipeline) + + if cfg.TargetVersion == "" { + // The dedicated pipeline always writes both keys, so an empty target + // there is a pipeline bug and must fail, not skip. + require.False(s.T(), pipeline.Enabled, "rancher-upgrade pipeline config is missing %s.targetVersion", UpgradeInputConfigKey) + // Generic PIT legs never upgrade the server between phases; without + // an upgrade target the phases have nothing to assert, so skip. + s.T().Skipf("%s.targetVersion is not set: this suite runs only in the rancher-upgrade pipeline, which upgrades the server between phases", UpgradeInputConfigKey) + } + + testSession := session.NewSession() + s.session = testSession + + client, err := rancher.NewClient("", testSession) + require.NoError(s.T(), err) + s.client = client + + s.cluster, err = clusters.NewClusterMeta(client, client.RancherConfig.ClusterName) + require.NoError(s.T(), err) +} + +// TearDownSuite deliberately does not call session.Cleanup(): resources created in +// the pre-upgrade phase (NeuVector chart, admission control deny rule and its deny +// namespace) must survive into the post-upgrade and chart-upgrade phases, which +// run as separate go test processes. The pipeline environment teardown is the +// cleanup for these resources. +func (s *NeuVectorUpgradeTestSuite) TearDownSuite() { +} + +func (s *NeuVectorUpgradeTestSuite) TestNeuVectorPreUpgrade() { + logrus.Infof("pre-upgrade phase: resolving the System project on cluster %s", s.cluster.ID) + project, err := nvSystemProject(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + // Resolve the version from the cluster-scoped catalog and install that: the + // management view can offer versions the cluster-scoped shard does not serve + // yet (rancher-2.15-catalog-sharding-issue.md), and the chart action validates + // against the cluster-scoped index. The pre-phase still yields the 2.14-era + // version by construction, since it runs against the pre-upgrade Rancher. + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + version, err := catalogClient.GetLatestChartVersion(charts.NeuVectorChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + defaultRegistry, err := nvSystemDefaultRegistry(s.client) + require.NoError(s.T(), err) + + logrus.Infof("pre-upgrade phase: installing neuvector %s from the %s repo", version, catalog.RancherChartRepo) + err = charts.InstallNeuVectorChart(s.client, charts.PayloadOpts{ + Namespace: charts.NeuVectorNamespace, + Host: s.client.RancherConfig.Host, + InstallOptions: charts.InstallOptions{ + Cluster: s.cluster, + Version: version, + ProjectID: project.ID, + }, + DefaultRegistry: defaultRegistry, + K3s: s.cluster.Provider == clusters.KubernetesProviderK3S, + }) + require.NoError(s.T(), err) + + logrus.Infof("pre-upgrade phase: neuvector %s installed; waiting for its workloads", version) + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return shepherdCharts.WaitChartInstall(catalogClient, charts.NeuVectorNamespace, charts.NeuVectorChartName) + }) + require.NoError(s.T(), err) + + err = shepherdCharts.WatchAndWaitDeployments(s.client, s.cluster.ID, charts.NeuVectorNamespace, metav1.ListOptions{}) + require.NoError(s.T(), err) + err = shepherdCharts.WatchAndWaitDaemonSets(s.client, s.cluster.ID, charts.NeuVectorNamespace, metav1.ListOptions{}) + require.NoError(s.T(), err) + + logrus.Infof("pre-upgrade phase: enabling the admission control deny rule for namespace %s", nvAdmissionDenyNamespace) + err = neuvector.EnableAdmissionControlDenyNamespace(s.client, s.cluster.ID, nvAdmissionDenyNamespace) + require.NoError(s.T(), err) + + err = neuvector.VerifyAdmissionDenial(s.client, s.cluster.ID, nvAdmissionDenyNamespace) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: admission denial proven in %s; pre-upgrade state seeded", nvAdmissionDenyNamespace) +} + +func (s *NeuVectorUpgradeTestSuite) TestNeuVectorPostUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("post-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: server version %s observed; checking neuvector survival", cfg.TargetVersion) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.NeuVectorNamespace, charts.NeuVectorChartName) + require.NoError(s.T(), err) + require.True(s.T(), chart.IsAlreadyInstalled, "neuvector chart must still be installed after the Rancher server upgrade") + logrus.Infof("post-upgrade phase: neuvector chart still installed; verifying services and manager UI") + + err = validationCharts.VerifyNeuVectorServicesActive(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + respUI, err := validationCharts.VerifyNeuVectorWebUIAccessible(s.client, s.cluster.ID) + require.NoError(s.T(), err, "neuvector manager web UI should be reachable via the Rancher service proxy") + require.NotEmpty(s.T(), respUI, "expected a non-empty response from the neuvector manager web UI") + logrus.Infof("post-upgrade phase: services active and manager UI reachable; verifying admission enforcement") + + err = neuvector.VerifyAdmissionDenial(s.client, s.cluster.ID, nvAdmissionDenyNamespace) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: admission webhook still denies creates in %s", nvAdmissionDenyNamespace) +} + +func (s *NeuVectorUpgradeTestSuite) TestNeuVectorChartUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("chart-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + + // Resolve the latest version from the cluster-scoped catalog: the management + // view can report versions absent from the cluster-scoped index (see + // rancher-2.15-catalog-sharding-issue.md at the repo root). + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + latest, err := catalogClient.GetLatestChartVersion(charts.NeuVectorChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + // Guard against a stale cluster-scoped catalog: until the cluster agent + // re-syncs after the server upgrade, the follower can still serve the + // previous Rancher line's index, whose latest equals the installed version. + // Upgrading to it would pass vacuously (a same-version upgrade asserted + // equal to itself), so fail with the runbook remedy instead; see README Notes. + installed, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.NeuVectorNamespace, charts.NeuVectorChartName) + require.NoError(s.T(), err) + require.NotEqual(s.T(), + installed.ChartDetails.Spec.Chart.Metadata.Version, latest, + "cluster-scoped catalog still serves the pre-upgrade index: installed neuvector %s equals latest %s; restart the cluster agent and re-run this phase", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + logrus.Infof("chart-upgrade phase: upgrading neuvector %s to %s from the upgraded catalog", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + + installOptions := &charts.InstallOptions{ + Cluster: s.cluster, + Version: latest, + } + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return charts.UpgradeNeuVectorChart(s.client, installOptions) + }) + require.NoError(s.T(), err) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.NeuVectorNamespace, charts.NeuVectorChartName) + require.NoError(s.T(), err) + require.Equal(s.T(), latest, chart.ChartDetails.Spec.Chart.Metadata.Version) + logrus.Infof("chart-upgrade phase: neuvector at %s; verifying health and enforcement", latest) + + err = validationCharts.VerifyNeuVectorServicesActive(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + err = neuvector.VerifyAdmissionDenial(s.client, s.cluster.ID, nvAdmissionDenyNamespace) + require.NoError(s.T(), err) + logrus.Infof("chart-upgrade phase: admission webhook still denies creates in %s", nvAdmissionDenyNamespace) +} + +func TestNeuVectorUpgradeTestSuite(t *testing.T) { + suite.Run(t, new(NeuVectorUpgradeTestSuite)) +} diff --git a/validation/rancherupgrade/schemas/pit_schemas.yaml b/validation/rancherupgrade/schemas/pit_schemas.yaml index a95dc71f59..e083e0d503 100644 --- a/validation/rancherupgrade/schemas/pit_schemas.yaml +++ b/validation/rancherupgrade/schemas/pit_schemas.yaml @@ -59,3 +59,73 @@ position: 4 custom_field: "15": "TestLonghornChartUpgrade" + - title: "NeuVector pre-upgrade setup" + description: "Install NeuVector and prove admission control enforcement before the Rancher server upgrade" + automation: 2 + steps: + - action: "Resolve the System project on the Rancher local cluster" + data: "Project: System" + expectedresult: "The System project is resolved for the chart install" + position: 1 + - action: "Install the NeuVector chart at the latest version offered by the pre-upgrade Rancher" + data: "Chart: neuvector and neuvector-crd from the rancher-charts repo in cattle-neuvector-system" + expectedresult: "NeuVector installs and its deployments and daemonsets become Ready" + position: 2 + - action: "Enable the NeuVector admission control deny rule" + data: "NvAdmissionControlSecurityRule named local in protect mode denying workload creates in namespace rancher-upgrade-neuvector-deny" + expectedresult: "The custom resource is applied and the neuvector-validating-admission-webhook ValidatingWebhookConfiguration exists" + position: 3 + - action: "Prove the admission webhook denies a known-bad create" + data: "Pod create in namespace rancher-upgrade-neuvector-deny" + expectedresult: "The create is rejected with an admission webhook denial" + position: 4 + custom_field: + "15": "TestNeuVectorPreUpgrade" + - title: "NeuVector post-upgrade validation" + description: "Verify NeuVector services, manager UI and admission enforcement survive the Rancher server upgrade" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Check the NeuVector chart installation state" + data: "Chart: neuvector in cattle-neuvector-system" + expectedresult: "The chart is still installed" + position: 2 + - action: "Verify the NeuVector services are active" + data: "neuvector-service-webui, neuvector-svc-controller, neuvector-svc-admission-webhook" + expectedresult: "All expected NeuVector services exist and are active" + position: 3 + - action: "Verify the NeuVector manager web UI is reachable" + data: "Rancher service proxy path for neuvector-service-webui port 8443" + expectedresult: "The manager web UI responds through the Rancher service proxy" + position: 4 + - action: "Verify the admission webhook still denies the known-bad create" + data: "Pod create in namespace rancher-upgrade-neuvector-deny" + expectedresult: "The create is rejected with an admission webhook denial" + position: 5 + custom_field: + "15": "TestNeuVectorPostUpgrade" + - title: "NeuVector chart upgrade after Rancher upgrade" + description: "Upgrade NeuVector to the version offered by the upgraded Rancher catalog and re-verify health and enforcement" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Resolve the latest NeuVector chart version from the cluster-scoped catalog" + data: "Chart: neuvector from the rancher-charts repo" + expectedresult: "The latest cluster-scoped chart version is resolved and differs from the installed pre-upgrade version" + position: 2 + - action: "Upgrade the NeuVector chart with retry, preserving the installed release values" + data: "neuvector and neuvector-crd to the latest version, reusing the installed App values" + expectedresult: "The neuvector chart reaches Deployed state and workloads become Ready" + position: 3 + - action: "Verify the chart version, service health and admission enforcement" + data: "Chart version equals latest; NeuVector services active; pod create in namespace rancher-upgrade-neuvector-deny denied" + expectedresult: "Installed chart version matches latest, all expected services are active and the admission webhook still denies the create" + position: 4 + custom_field: + "15": "TestNeuVectorChartUpgrade" From 4532b39e4f6df57419184f77e95852ad10939ffc Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 22 Oct 2026 08:37:00 -0500 Subject: [PATCH 17/19] Fix pit_schemas: give NeuVector cases their own suite document --- validation/rancherupgrade/schemas/pit_schemas.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/validation/rancherupgrade/schemas/pit_schemas.yaml b/validation/rancherupgrade/schemas/pit_schemas.yaml index e083e0d503..fea4abbc2d 100644 --- a/validation/rancherupgrade/schemas/pit_schemas.yaml +++ b/validation/rancherupgrade/schemas/pit_schemas.yaml @@ -59,6 +59,10 @@ position: 4 custom_field: "15": "TestLonghornChartUpgrade" + +- projects: [RANCHERINT] + suite: Rancher Upgrade/NeuVector + cases: - title: "NeuVector pre-upgrade setup" description: "Install NeuVector and prove admission control enforcement before the Rancher server upgrade" automation: 2 From 9802caddcf5c4a5d5b685684f04b2fd42f36e3db Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Thu, 22 Oct 2026 10:22:59 -0500 Subject: [PATCH 18/19] Treat Steve plain-text 404s as not-found in admission helpers The Steve client returns errors like "Bad response statusCode [404]. Status [404 Not Found]..." that k8sErrors.IsNotFound does not match, so on a fresh cluster the deny-namespace lookup, the admission CR lookup and the webhook poll all surfaced their 404s instead of taking the create/wait paths (seen live: TestNeuVectorPreUpgrade failed with namespaces "rancher-upgrade-neuvector-deny" not found). Route all four checks through isNotFound, which also matches the Steve error text. --- actions/neuvector/admission.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/actions/neuvector/admission.go b/actions/neuvector/admission.go index 58eda115d4..fdbce91fe3 100644 --- a/actions/neuvector/admission.go +++ b/actions/neuvector/admission.go @@ -136,7 +136,7 @@ func applyAdmissionControlRule(steveClient *v1.Client, denyNamespace string) err existing, err := crClient.ByID(AdmissionControlCRName) if err != nil { - if !k8sErrors.IsNotFound(err) { + if !isNotFound(err) { return err } @@ -198,7 +198,7 @@ func waitForAdmissionWebhook(steveClient *v1.Client) error { return kwait.PollUntilContextTimeout(context.Background(), admissionPollInterval, admissionPollTimeout, true, func(ctx context.Context) (bool, error) { _, err := webhookClient.ByID(NeuVectorValidatingWebhookName) if err != nil { - if k8sErrors.IsNotFound(err) { + if isNotFound(err) { return false, nil } return false, err @@ -217,7 +217,7 @@ func ensureNamespace(steveClient *v1.Client, name string) error { if err == nil { return nil } - if !k8sErrors.IsNotFound(err) { + if !isNotFound(err) { return err } @@ -262,7 +262,7 @@ func attemptAdmissionProbeCreate(steveClient *v1.Client, namespace string) error func deleteAdmissionProbePod(steveClient *v1.Client, namespace string) error { probe, err := steveClient.SteveType(stevetypes.Pod).ByID(namespace + "/" + admissionProbePodName) if err != nil { - if k8sErrors.IsNotFound(err) { + if isNotFound(err) { return nil } return err @@ -276,3 +276,17 @@ func deleteAdmissionProbePod(steveClient *v1.Client, namespace string) error { func isAdmissionWebhookDenial(err error) bool { return err != nil && strings.Contains(strings.ToLower(err.Error()), admissionWebhookDenialMarker) } + +// isNotFound reports whether err is a resource-not-found error from either +// client type: the Kubernetes client returns a typed StatusError that +// k8sErrors.IsNotFound matches, while the Steve client returns plain-text +// errors carrying the HTTP status line (for example "Bad response statusCode +// [404]. Status [404 Not Found]. Body: [code=NotFound, message=...]"), which +// IsNotFound does not match and which must be detected textually. +func isNotFound(err error) bool { + if err == nil || k8sErrors.IsNotFound(err) { + return err != nil + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "404") && strings.Contains(msg, "not found") +} From 8eba49a848190e0b39094247437bfa7046f0f7ce Mon Sep 17 00:00:00 2001 From: Daniel Newman Date: Wed, 28 Oct 2026 14:59:17 -0500 Subject: [PATCH 19/19] Rename neuvector_test.go to neuvector_upgrade_test.go Consistent module-upgrade file naming across the rancherupgrade suites (review follow-up, khushboo-rancher); the file moves with its suite, which was already named TestNeuVectorUpgradeTestSuite. --- .../{neuvector_test.go => neuvector_upgrade_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename validation/rancherupgrade/{neuvector_test.go => neuvector_upgrade_test.go} (100%) diff --git a/validation/rancherupgrade/neuvector_test.go b/validation/rancherupgrade/neuvector_upgrade_test.go similarity index 100% rename from validation/rancherupgrade/neuvector_test.go rename to validation/rancherupgrade/neuvector_upgrade_test.go