diff --git a/actions/charts/longhorn.go b/actions/charts/longhorn.go index 89c8640cb3..c1191e3859 100644 --- a/actions/charts/longhorn.go +++ b/actions/charts/longhorn.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" "github.com/rancher/shepherd/clients/rancher" "github.com/rancher/shepherd/clients/rancher/catalog" shepherdCharts "github.com/rancher/shepherd/extensions/charts" @@ -100,6 +101,123 @@ func InstallLonghornChart(client *rancher.Client, payload PayloadOpts, values ma return err } +// UpgradeLonghornChart upgrades the longhorn and longhorn-crd charts to +// installOptions.Version on installOptions.Cluster and waits for Deployed state. +// No session cleanup is registered: the uninstall cleanup belongs to the install +// call of the same session, and upgrade-test sessions must not delete the chart. +func UpgradeLonghornChart(client *rancher.Client, installOptions *InstallOptions) error { + serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) + if err != nil { + return err + } + + registrySetting, err := client.Management.Setting.ByID(defaultRegistrySettingID) + if err != nil { + return err + } + + longhornChartUpgradeActionPayload := &PayloadOpts{ + InstallOptions: *installOptions, + Name: LonghornChartName, + Namespace: LonghornNamespace, + Host: serverSetting.Value, + DefaultRegistry: registrySetting.Value, + } + + chartUpgradeAction := newLonghornChartUpgradeAction(longhornChartUpgradeActionPayload) + + catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + bodyBytes, err := json.Marshal(chartUpgradeAction) + if err != nil { + return err + } + err = ChartActionWithRetry(context.TODO(), client, verbUpgrade, longhornChartUpgradeActionPayload, catalog.RancherChartRepo, []string{LonghornChartName, LonghornChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbUpgrade, bodyBytes)) + if err != nil { + return err + } + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + if err != nil { + return err + } + + adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + watchAppInterface, err := adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + LonghornChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusPendingUpgrade) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + watchAppInterface, err = adminCatalogClient.Apps(LonghornNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + LonghornChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusDeployed) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{}) + if err != nil { + return err + } + + return shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, LonghornNamespace, metav1.ListOptions{}) +} + +// newLonghornChartUpgradeAction is a private helper function that returns the chart upgrade action for Longhorn. +func newLonghornChartUpgradeAction(p *PayloadOpts) *types.ChartUpgradeAction { + // Pass the deletingConfirmationFlag value on the main chart so the upgrade does + // not reset it; the CRD chart takes no Longhorn settings (same split as install). + chartUpgrade := NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, enableDeletionSetting) + chartUpgradeCRD := NewChartUpgrade(p.Name+"-crd", p.Name+"-crd", p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, nil) + chartUpgrades := []types.ChartUpgrade{*chartUpgradeCRD, *chartUpgrade} + + chartUpgradeAction := NewChartUpgradeAction(p.Namespace, chartUpgrades) + // Disable OpenAPI validation to avoid schema-conflict failures on the large + // Longhorn CRDs, which frequently change shape between versions. + chartUpgradeAction.DisableOpenAPIValidation = true + + return chartUpgradeAction +} + // UninstallLonghornChart removes Longhorn from the cluster related to the received catalog client object. func UninstallLonghornChart(client *rancher.Client, namespace string, clusterID string, rancherHostname string) error { catalogClient, err := client.GetClusterCatalogClient(clusterID) diff --git a/actions/charts/neuvector.go b/actions/charts/neuvector.go index 28786b1563..b94c0f0321 100644 --- a/actions/charts/neuvector.go +++ b/actions/charts/neuvector.go @@ -5,13 +5,18 @@ import ( "encoding/json" "time" + catalogv1 "github.com/rancher/rancher/pkg/apis/catalog.cattle.io/v1" "github.com/rancher/shepherd/clients/rancher" "github.com/rancher/shepherd/clients/rancher/catalog" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/defaults" "github.com/rancher/shepherd/pkg/api/steve/catalog/types" + "github.com/rancher/shepherd/pkg/wait" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kwait "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" ) const ( @@ -178,6 +183,137 @@ func InstallNeuVectorChart(client *rancher.Client, payload PayloadOpts) error { return nil } +// UpgradeNeuVectorChart upgrades the neuvector and neuvector-crd charts to +// installOptions.Version on installOptions.Cluster, waits for the Deployed state +// and for the NeuVector workloads (controller deployment, enforcer daemonset) to +// become Ready again. +// +// The upgrade carries over the values of the installed release (the App resource's +// spec.values, equivalent to helm --reuse-values) instead of applying a blind +// version bump: registry overrides required by airgap environments and the +// enforcer hardening or k3s options set at install time must survive the upgrade, +// or the upgraded NeuVector would come back with chart defaults (docker.io +// images, unhardened enforcer) and break those environments. The neuvector-crd +// chart takes no values, mirroring the install split. +// +// No session cleanup is registered: the uninstall cleanup belongs to the install +// call of the same session, and upgrade-test sessions must not delete the chart. +func UpgradeNeuVectorChart(client *rancher.Client, installOptions *InstallOptions) error { + serverSetting, err := client.Management.Setting.ByID(serverURLSettingID) + if err != nil { + return err + } + + registrySetting, err := client.Management.Setting.ByID(defaultRegistrySettingID) + if err != nil { + return err + } + + catalogClient, err := client.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + // Reading the installed App yields exactly the values overridden at install + // time (chart defaults are not stored), which is what must be re-applied. + installedApp, err := catalogClient.Apps(NeuVectorNamespace).Get(context.TODO(), NeuVectorChartName, metav1.GetOptions{}) + if err != nil { + return err + } + + neuVectorChartUpgradeActionPayload := &PayloadOpts{ + InstallOptions: *installOptions, + Name: NeuVectorChartName, + Namespace: NeuVectorNamespace, + Host: serverSetting.Value, + DefaultRegistry: registrySetting.Value, + } + + chartUpgradeAction := newNeuVectorChartUpgradeAction(neuVectorChartUpgradeActionPayload, map[string]interface{}(installedApp.Spec.Values)) + + bodyBytes, err := json.Marshal(chartUpgradeAction) + if err != nil { + return err + } + err = ChartActionWithRetry(context.TODO(), client, verbUpgrade, neuVectorChartUpgradeActionPayload, catalog.RancherChartRepo, []string{NeuVectorChartName, NeuVectorChartName + "-crd"}, buildRepoActionRequest(catalogClient, catalog.RancherChartRepo, verbUpgrade, bodyBytes)) + if err != nil { + return err + } + + adminClient, err := rancher.NewClient(client.RancherConfig.AdminToken, client.Session) + if err != nil { + return err + } + + adminCatalogClient, err := adminClient.GetClusterCatalogClient(installOptions.Cluster.ID) + if err != nil { + return err + } + + watchAppInterface, err := adminCatalogClient.Apps(NeuVectorNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + NeuVectorChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusPendingUpgrade) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + watchAppInterface, err = adminCatalogClient.Apps(NeuVectorNamespace).Watch(context.TODO(), metav1.ListOptions{ + FieldSelector: "metadata.name=" + NeuVectorChartName, + TimeoutSeconds: &defaults.WatchTimeoutSeconds, + }) + if err != nil { + return err + } + + err = wait.WatchWait(watchAppInterface, func(event watch.Event) (ready bool, err error) { + app := event.Object.(*catalogv1.App) + + state := app.Status.Summary.State + if state == string(catalogv1.StatusDeployed) { + return true, nil + } + + return false, nil + }) + if err != nil { + return err + } + + err = shepherdCharts.WatchAndWaitDeployments(client, installOptions.Cluster.ID, NeuVectorNamespace, metav1.ListOptions{}) + if err != nil { + return err + } + + return shepherdCharts.WatchAndWaitDaemonSets(client, installOptions.Cluster.ID, NeuVectorNamespace, metav1.ListOptions{}) +} + +// newNeuVectorChartUpgradeAction is a private helper function that returns the +// chart upgrade action for NeuVector, reusing the installed release values on the +// main chart. OpenAPI validation stays enabled: the neuvector CRDs are small and +// do not hit the schema-conflict failures seen on large CRD charts. +func newNeuVectorChartUpgradeAction(p *PayloadOpts, installedValues map[string]interface{}) *types.ChartUpgradeAction { + chartUpgrade := NewChartUpgrade(p.Name, p.Name, p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, installedValues) + chartUpgradeCRD := NewChartUpgrade(p.Name+"-crd", p.Name+"-crd", p.Version, p.Cluster.ID, p.Cluster.Name, p.Host, p.DefaultRegistry, nil) + chartUpgrades := []types.ChartUpgrade{*chartUpgradeCRD, *chartUpgrade} + + return NewChartUpgradeAction(p.Namespace, chartUpgrades) +} + func uninstallNeuVectorChart(client *rancher.Client, namespace string, clusterID string) error { catalogClient, err := client.GetClusterCatalogClient(clusterID) if err != nil { diff --git a/actions/clusters/local.go b/actions/clusters/local.go index 206e523b81..16940dfd53 100644 --- a/actions/clusters/local.go +++ b/actions/clusters/local.go @@ -1,13 +1,18 @@ package clusters import ( + "context" "errors" "fmt" "regexp" "strings" + "time" "github.com/Masterminds/semver/v3" "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/wait" ) const ( @@ -37,6 +42,41 @@ func IsRancherVersionAbove(client *rancher.Client, minVersion string) (bool, err return serverVersion.GreaterThan(minimumVersion), nil } +// WaitRancherVersion polls the management server-version setting until it equals +// targetVersion (semver-compared) or timeout elapses. Gates post-upgrade assertions +// on Rancher server convergence; transient read errors during the upgrade rollout +// are retried, not returned. +func WaitRancherVersion(client *rancher.Client, targetVersion string, timeout time.Duration) error { + target, err := parseRancherVersion(targetVersion) + if err != nil { + return err + } + + return wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, timeout, true, func(ctx context.Context) (done bool, err error) { + serverVersionRawValue, err := client.Management.Setting.ByID(serverVersionSetting) + if err != nil { + // The server may be mid-restart during the upgrade rollout; retry the read. + // Logged so a terminal cause (e.g. a revoked token or unreachable API) + // is visible in the log instead of surfacing only as the final timeout. + logrus.Debugf("server-version read failed, retrying: %v", err) + return false, nil + } + + serverVersion, err := parseRancherVersion(serverVersionRawValue.Value) + if err != nil { + logrus.Debugf("server-version value %q not parseable yet, retrying: %v", serverVersionRawValue.Value, err) + return false, nil + } + + if serverVersion.Equal(target) { + return true, nil + } + + logrus.Debugf("Rancher server version is %q, waiting for %q", serverVersionRawValue.Value, targetVersion) + return false, nil + }) +} + func parseRancherVersion(versionInput string) (*semver.Version, error) { trimmedVersion := strings.TrimSpace(strings.TrimPrefix(versionInput, "v")) if trimmedVersion == "" { diff --git a/actions/neuvector/admission.go b/actions/neuvector/admission.go new file mode 100644 index 0000000000..fdbce91fe3 --- /dev/null +++ b/actions/neuvector/admission.go @@ -0,0 +1,292 @@ +// Package neuvector provides reusable helpers around NeuVector on Rancher-managed +// clusters: test configuration and admission-control enforcement exercised by the +// validation suites. +package neuvector + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/rancher/shepherd/clients/rancher" + v1 "github.com/rancher/shepherd/clients/rancher/v1" + "github.com/rancher/shepherd/extensions/defaults/stevetypes" + "github.com/rancher/tests/actions/namespaces" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kwait "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + // AdmissionControlSteveType is the steve type of the cluster-scoped + // NvAdmissionControlSecurityRule custom resource served by the + // neuvector-crd chart (group neuvector.com, singular nvadmissioncontrolsecurityrule). + AdmissionControlSteveType = "neuvector.com.nvadmissioncontrolsecurityrule" + + // AdmissionControlCRName is the resource name the NeuVector CRD documentation + // requires for the admission control custom resource: always "local". + AdmissionControlCRName = "local" + + // ValidatingWebhookSteveType is the steve type of the Kubernetes + // ValidatingWebhookConfiguration resource on the downstream cluster. + ValidatingWebhookSteveType = "admissionregistration.k8s.io.validatingwebhookconfiguration" + + // NeuVectorValidatingWebhookName is the ValidatingWebhookConfiguration NeuVector + // registers automatically once admission control is enabled. + NeuVectorValidatingWebhookName = "neuvector-validating-admission-webhook" + + // admissionWebhookDenialMarker is the substring a Kubernetes API server error + // contains when a create is rejected by an admission webhook; matched + // case-insensitively. + admissionWebhookDenialMarker = "admission webhook" + + admissionPollInterval = 5 * time.Second + admissionPollTimeout = 3 * time.Minute + + // admissionProbePodName is the deterministic name of the pod whose creation + // the admission deny rule must reject; it never starts running, so the name + // stays free for every verification attempt. + admissionProbePodName = "rancher-admission-deny-probe" + + // admissionProbePodImage is only referenced by the denied create request and + // is never pulled while the webhook enforces the deny rule. + admissionProbePodImage = "busybox:1.36" + + admissionProbeContainerName = "deny-probe" +) + +// EnableAdmissionControlDenyNamespace enables NeuVector admission control on the +// cluster and applies a protect-mode deny rule rejecting any workload create in +// denyNamespace. The NvAdmissionControlSecurityRule custom resource is created +// (or updated) under the NeuVector-documented name "local" with the shape of the +// console-exported manifest: config enable/protect/service plus a deny rule whose +// namespace criterion uses the containsAny operator. The deny namespace is created +// if missing, and the call returns only after NeuVector has registered its +// validating webhook, so enforcement can be proven right away with +// VerifyAdmissionDenial. Everything is named deterministically and nothing is +// registered for session cleanup: upgrade-test phases must keep the rule in force +// across separate go test processes. +func EnableAdmissionControlDenyNamespace(client *rancher.Client, clusterID, denyNamespace string) error { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return err + } + + if err := ensureNamespace(steveClient, denyNamespace); err != nil { + return err + } + + if err := applyAdmissionControlRule(steveClient, denyNamespace); err != nil { + return err + } + + return waitForAdmissionWebhook(steveClient) +} + +// VerifyAdmissionDenial proves the NeuVector admission webhook enforces the deny +// rule for denyNamespace: it attempts to create a pod there and expects the API +// server to reject the create with an admission-webhook denial. Attempts are +// retried within a bounded window because the NeuVector controllers reload the +// CRD rules asynchronously (for example while rolling after a chart upgrade), and +// a pod created before the rule is loaded again is deleted before the next +// attempt. An unexpected create success that outlives the window, or a persistent +// non-webhook error, is returned as an error. +func VerifyAdmissionDenial(client *rancher.Client, clusterID, denyNamespace string) error { + steveClient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return err + } + + var lastErr error + err = kwait.PollUntilContextTimeout(context.Background(), admissionPollInterval, admissionPollTimeout, true, func(ctx context.Context) (bool, error) { + lastErr = attemptAdmissionProbeCreate(steveClient, denyNamespace) + switch { + case lastErr == nil: + // The deny rule did not fire: NeuVector may still be loading the CRD + // rules after a restart, so remove the stray pod and retry. + logrus.Warnf("admission probe pod in namespace %s was created instead of denied; deleting it and retrying", denyNamespace) + if deleteErr := deleteAdmissionProbePod(steveClient, denyNamespace); deleteErr != nil { + return false, deleteErr + } + lastErr = fmt.Errorf("pod %s/%s was created but the NeuVector admission webhook must deny workload creates in this namespace", denyNamespace, admissionProbePodName) + return false, nil + case isAdmissionWebhookDenial(lastErr): + logrus.Infof("NeuVector admission webhook denied the probe create in namespace %s as expected", denyNamespace) + return true, nil + default: + // Retry transient API errors, but keep the surprise for the timeout. + logrus.Warnf("probe create in namespace %s failed with a non-webhook error, retrying: %v", denyNamespace, lastErr) + return false, nil + } + }) + if err != nil { + return fmt.Errorf("NeuVector admission enforcement not proven for namespace %s: %w", denyNamespace, lastErr) + } + + return nil +} + +// applyAdmissionControlRule creates or updates the NvAdmissionControlSecurityRule +// named "local" with the deny rule for denyNamespace. +func applyAdmissionControlRule(steveClient *v1.Client, denyNamespace string) error { + crClient := steveClient.SteveType(AdmissionControlSteveType) + + existing, err := crClient.ByID(AdmissionControlCRName) + if err != nil { + if !isNotFound(err) { + return err + } + + cr := &v1.SteveAPIObject{} + cr.TypeMeta = metav1.TypeMeta{Kind: "NvAdmissionControlSecurityRule", APIVersion: "neuvector.com/v1"} + cr.Name = AdmissionControlCRName + cr.Spec = admissionControlSpec(denyNamespace) + + logrus.Infof("creating NeuVector admission control rule denying workload creates in namespace %s", denyNamespace) + _, err = crClient.Create(cr) + return err + } + + updated := *existing + updated.Spec = admissionControlSpec(denyNamespace) + + logrus.Infof("updating NeuVector admission control rule denying workload creates in namespace %s", denyNamespace) + _, err = crClient.Update(existing, &updated) + return err +} + +// admissionControlSpec builds the spec of the admission control custom resource +// following the console-exported manifest documented by NeuVector: admission +// control enabled in protect mode through the service client, plus a deny rule +// matching the deny namespace with the containsAny operator on the namespace +// criterion. Protect mode is what turns the deny action into an inline rejection. +func admissionControlSpec(denyNamespace string) map[string]any { + return map[string]any{ + "config": map[string]any{ + "enable": true, + "mode": "protect", + "client_mode": "service", + }, + "rules": []any{ + map[string]any{ + "action": "deny", + "containers": []any{"containers"}, + "criteria": []any{ + map[string]any{ + "name": "namespace", + "op": "containsAny", + "path": "namespace", + "value": denyNamespace, + }, + }, + "disabled": false, + "rule_mode": "", + }, + }, + } +} + +// waitForAdmissionWebhook polls for the ValidatingWebhookConfiguration NeuVector +// registers once admission control is enabled; until it exists the API server +// cannot consult NeuVector at all. +func waitForAdmissionWebhook(steveClient *v1.Client) error { + webhookClient := steveClient.SteveType(ValidatingWebhookSteveType) + + return kwait.PollUntilContextTimeout(context.Background(), admissionPollInterval, admissionPollTimeout, true, func(ctx context.Context) (bool, error) { + _, err := webhookClient.ByID(NeuVectorValidatingWebhookName) + if err != nil { + if isNotFound(err) { + return false, nil + } + return false, err + } + + return true, nil + }) +} + +// ensureNamespace creates the namespace if it does not exist yet; a namespace +// create is not a workload create, so the admission deny rule does not reject it. +func ensureNamespace(steveClient *v1.Client, name string) error { + nsClient := steveClient.SteveType(namespaces.NamespaceSteveType) + + _, err := nsClient.ByID(name) + if err == nil { + return nil + } + if !isNotFound(err) { + return err + } + + ns := &v1.SteveAPIObject{} + ns.TypeMeta = metav1.TypeMeta{Kind: "Namespace", APIVersion: "v1"} + ns.Name = name + + logrus.Infof("creating namespace %s for the admission deny probe", name) + if _, err = nsClient.Create(ns); err != nil { + // Tolerate a create race by re-checking existence. + if _, getErr := nsClient.ByID(name); getErr != nil { + return err + } + } + + return nil +} + +// attemptAdmissionProbeCreate tries to create the probe pod whose name and image +// are fixed; a nil return means the create was accepted. +func attemptAdmissionProbeCreate(steveClient *v1.Client, namespace string) error { + probe := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: admissionProbePodName, + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: admissionProbeContainerName, + Image: admissionProbePodImage, + }, + }, + }, + } + + _, err := steveClient.SteveType(stevetypes.Pod).Create(probe) + return err +} + +// deleteAdmissionProbePod removes a probe pod that escaped the deny rule. +func deleteAdmissionProbePod(steveClient *v1.Client, namespace string) error { + probe, err := steveClient.SteveType(stevetypes.Pod).ByID(namespace + "/" + admissionProbePodName) + if err != nil { + if isNotFound(err) { + return nil + } + return err + } + + return steveClient.SteveType(stevetypes.Pod).Delete(probe) +} + +// isAdmissionWebhookDenial reports whether err is a Kubernetes admission-webhook +// rejection, the error shape the API server returns for a denied create. +func isAdmissionWebhookDenial(err error) bool { + return err != nil && strings.Contains(strings.ToLower(err.Error()), admissionWebhookDenialMarker) +} + +// isNotFound reports whether err is a resource-not-found error from either +// client type: the Kubernetes client returns a typed StatusError that +// k8sErrors.IsNotFound matches, while the Steve client returns plain-text +// errors carrying the HTTP status line (for example "Bad response statusCode +// [404]. Status [404 Not Found]. Body: [code=NotFound, message=...]"), which +// IsNotFound does not match and which must be detected textually. +func isNotFound(err error) bool { + if err == nil || k8sErrors.IsNotFound(err) { + return err != nil + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "404") && strings.Contains(msg, "not found") +} diff --git a/actions/storage/checksum.go b/actions/storage/checksum.go new file mode 100644 index 0000000000..e7da7d5b29 --- /dev/null +++ b/actions/storage/checksum.go @@ -0,0 +1,221 @@ +package storage + +import ( + "context" + "crypto/sha256" + "fmt" + "regexp" + "strings" + + "github.com/rancher/shepherd/clients/rancher" + steveV1 "github.com/rancher/shepherd/clients/rancher/v1" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/extensions/defaults/namespaces" + "github.com/rancher/shepherd/extensions/defaults/stevetypes" + "github.com/rancher/shepherd/extensions/kubeapi/cluster" + "github.com/rancher/shepherd/extensions/kubeconfig" + "github.com/rancher/tests/actions/kubeapi/volumes/persistentvolumeclaims" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" +) + +// ChecksumDataFile is the filename written under MountPath by CreateChecksumWorkload. +const ChecksumDataFile = "checksum-data" + +// checksumContentPattern constrains the content accepted by CreateChecksumWorkload: +// content is interpolated into a single-quoted sh -c command, so a value outside +// alphanumeric/hyphen (notably a quote) would break the quoting. Enforced in code +// so a future caller cannot silently inject shell. +var checksumContentPattern = regexp.MustCompile(`^[A-Za-z0-9-]+$`) + +// CreateChecksumWorkload creates a 1GiB PVC named on storageClassName in the +// default namespace, mounts it into a single-replica nginx deployment named at +// MountPath, waits for rollout, then writes content to MountPath/ChecksumDataFile via +// kubectl exec. Deterministic names let a later test phase (running as a separate +// process) relocate the resources. content must be shell-safe (alphanumeric/hyphen); +// it is interpolated into an sh -c command. No session cleanup is registered: callers +// are upgrade tests whose resources must outlive the process. +func CreateChecksumWorkload(client *rancher.Client, clusterID string, storageClassName string, name string, content string) error { + if !checksumContentPattern.MatchString(content) { + return fmt.Errorf("content %q must match %s: it is interpolated into a single-quoted sh -c command", + content, checksumContentPattern.String()) + } + + storageClass, err := GetStorageClass(client, clusterID, storageClassName) + if err != nil { + return err + } + + wrangler, err := cluster.GetClusterWranglerContext(client, clusterID) + if err != nil { + return err + } + + accessModes := []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + } + + // Deliberate deviation from CreatePVC: no PV-delete/Longhorn-volume cleanup funcs + // are registered, because these resources must survive into later upgrade phases. + _, err = persistentvolumeclaims.CreatePersistentVolumeClaim( + client, + clusterID, + name, + "test-pvc-volume", + namespaces.Default, + 1, + accessModes, + nil, + &storageClass, + ) + if err != nil { + return err + } + + pvc := &corev1.PersistentVolumeClaim{} + // TenMinuteTimeout: the first volume on a freshly installed Longhorn waits + // on engine-image pull, instance-manager startup, and CSI registration, + // which routinely exceeds a minute on a cold cluster. + err = wait.PollUntilContextTimeout(context.Background(), pollInterval, defaults.TenMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + pvc, err = wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + + return pvc.Status.Phase == persistentvolumeclaims.PersistentVolumeBoundStatus, nil + }) + if err != nil { + return err + } + logrus.Debugf("PVC %s is Bound to volume %s", pvc.Name, pvc.Spec.VolumeName) + + // Shared nginx+PVC deployment construction with CreatePVCWorkload; the + // deterministic name makes PVC, volume mount, deployment, and relocation + // label all equal name, so later phases re-find the workload from a + // separate process. + _, err = createNginxDeploymentWithPVC(client, clusterID, name, name, name) + if err != nil { + return err + } + + pod, err := getChecksumPod(client, clusterID, name) + if err != nil { + return err + } + + restConfig, err := getDownstreamRestConfig(client, clusterID) + if err != nil { + return err + } + + writeCommand := []string{"sh", "-c", "printf '%s' '" + content + "' > " + MountPath + "/" + ChecksumDataFile} + logrus.Debugf("Writing %s/%s on pod %s", MountPath, ChecksumDataFile, pod.Name) + _, err = kubeconfig.KubectlExec(restConfig, pod.Name, namespaces.Default, writeCommand) + + return err +} + +// VerifyChecksumWorkload relocates the PVC and pod created by CreateChecksumWorkload, +// asserts the PVC is still Bound, and asserts the sha256 of MountPath/ChecksumDataFile +// inside the running pod equals sha256(content). +func VerifyChecksumWorkload(client *rancher.Client, clusterID string, name string, content string) error { + wrangler, err := cluster.GetClusterWranglerContext(client, clusterID) + if err != nil { + return err + } + + pvc, err := wrangler.Core.PersistentVolumeClaim().Get(namespaces.Default, name, metav1.GetOptions{}) + if err != nil { + return err + } + + if pvc.Status.Phase != persistentvolumeclaims.PersistentVolumeBoundStatus { + return fmt.Errorf("PVC %s is %s, expected %s", name, pvc.Status.Phase, persistentvolumeclaims.PersistentVolumeBoundStatus) + } + + pod, err := getChecksumPod(client, clusterID, name) + if err != nil { + return err + } + + restConfig, err := getDownstreamRestConfig(client, clusterID) + if err != nil { + return err + } + + checksumCommand := []string{"sha256sum", MountPath + "/" + ChecksumDataFile} + output, err := kubeconfig.KubectlExec(restConfig, pod.Name, namespaces.Default, checksumCommand) + if err != nil { + return err + } + + // The exec runs with a TTY, so the output carries \r line endings; strip them + // before extracting the hash (first whitespace-separated field). + outputFields := strings.Fields(strings.ReplaceAll(output.String(), "\r", "")) + if len(outputFields) == 0 { + return fmt.Errorf("empty sha256sum output on pod %s", pod.Name) + } + + actualChecksum := outputFields[0] + expectedChecksum := fmt.Sprintf("%x", sha256.Sum256([]byte(content))) + if actualChecksum != expectedChecksum { + return fmt.Errorf("checksum mismatch on pod %s: expected %s, got %s", pod.Name, expectedChecksum, actualChecksum) + } + + return nil +} + +// getChecksumPod returns the pod in the default namespace labeled with +// DeploymentIdentifierLabel == name, waiting until it exists and is Running. +func getChecksumPod(client *rancher.Client, clusterID string, name string) (*steveV1.SteveAPIObject, error) { + var checksumPod *steveV1.SteveAPIObject + err := wait.PollUntilContextTimeout(context.Background(), defaults.TenSecondTimeout, defaults.TwoMinuteTimeout, true, func(ctx context.Context) (done bool, err error) { + steveclient, err := client.Steve.ProxyDownstream(clusterID) + if err != nil { + return false, err + } + + podsResp, err := steveclient.SteveType(stevetypes.Pod).NamespacedSteveClient(namespaces.Default).List(nil) + if err != nil { + return false, err + } + + for i := range podsResp.Data { + pod := &podsResp.Data[i] + if pod.Labels[DeploymentIdentifierLabel] != name { + continue + } + + podStatus := &corev1.PodStatus{} + err = steveV1.ConvertToK8sType(pod.Status, podStatus) + if err != nil { + return false, err + } + + if podStatus.Phase != corev1.PodRunning { + logrus.Debugf("Pod %s is %s, waiting for Running", pod.Name, podStatus.Phase) + return false, nil + } + + checksumPod = pod + return true, nil + } + + return false, nil + }) + + return checksumPod, err +} + +// getDownstreamRestConfig builds a rest config for kubectl exec against the cluster. +func getDownstreamRestConfig(client *rancher.Client, clusterID string) (*rest.Config, error) { + kubeConfig, err := kubeconfig.GetKubeconfig(client, clusterID) + if err != nil { + return nil, err + } + + return (*kubeConfig).ClientConfig() +} diff --git a/actions/storage/pvc.go b/actions/storage/pvc.go index 163c0df025..4030e6823d 100644 --- a/actions/storage/pvc.go +++ b/actions/storage/pvc.go @@ -158,7 +158,6 @@ func CreatePVC(client *rancher.Client, clusterID string, storageClassName string } return false, err }) - if err != nil { return nil, "", err } @@ -185,19 +184,25 @@ func CreatePVCWorkload(t *testing.T, client *rancher.Client, clusterID string, s persistentVolumeClaim, volumeName, err := CreatePVC(client, clusterID, storageClassName) require.NoError(t, err) - logrus.Debugf("Creating Nginx deployment with %s PVC (volume name %s)", persistentVolumeClaim.Name, volumeName) - nginxResponse, err := createNginxDeploymentWithPVC(client, clusterID, persistentVolumeClaim.Name, volumeName) + // Random workload name: this helper targets single-process functional tests; + // callers needing a deterministic (cross-phase relocatable) name call + // createNginxDeploymentWithPVC directly (see CreateChecksumWorkload). + deploymentName := namegenerator.AppendRandomString(containerNamePrefix) + logrus.Debugf("Creating Nginx deployment %s with %s PVC (volume name %s)", deploymentName, persistentVolumeClaim.Name, volumeName) + nginxResponse, err := createNginxDeploymentWithPVC(client, clusterID, persistentVolumeClaim.Name, volumeName, deploymentName) require.NoError(t, err) return nginxResponse } -// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace -func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcName, volName string) (*steveV1.SteveAPIObject, error) { +// createNginxDeploymentWithPVC is a helper function that creates a nginx deployment in a cluster's default namespace. +// The deployment and its DeploymentIdentifierLabel both use deploymentName, letting callers +// pick a random name (functional tests) or a deterministic name (workloads a later test +// phase must re-find from a separate process). +func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcName, volName, deploymentName string) (*steveV1.SteveAPIObject, error) { logrus.Tracef("Vol: %s", volName) logrus.Tracef("Pod: %s", pvcName) - containerName := namegenerator.AppendRandomString(containerNamePrefix) volMount := &corev1.VolumeMount{ MountPath: MountPath, Name: volName, @@ -218,8 +223,8 @@ func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcN } containerTemplate := wloads.NewContainer(nginxName, nginxName, corev1.PullAlways, []corev1.VolumeMount{*volMount}, []corev1.EnvFromSource{}, nil, nil, nil) - podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: containerName}, nil) - deployment := wloads.NewDeploymentTemplate(containerName, namespaces.Default, podTemplate, true, nil) + podTemplate := wloads.NewPodTemplate([]corev1.Container{containerTemplate}, []corev1.Volume{podVol}, []corev1.LocalObjectReference{}, map[string]string{DeploymentIdentifierLabel: deploymentName}, nil) + deployment := wloads.NewDeploymentTemplate(deploymentName, namespaces.Default, podTemplate, true, nil) deploymentResp, err := steveclient.SteveType(stevetypes.Deployment).Create(deployment) if err != nil { @@ -227,7 +232,7 @@ func createNginxDeploymentWithPVC(client *rancher.Client, clusterID string, pvcN } err = charts.WatchAndWaitDeployments(client, clusterID, namespaces.Default, metav1.ListOptions{ - FieldSelector: "metadata.name=" + containerName, + FieldSelector: "metadata.name=" + deploymentName, }) return deploymentResp, err diff --git a/validation/Jenkinsfile.rancher-upgrade.e2e b/validation/Jenkinsfile.rancher-upgrade.e2e new file mode 100755 index 0000000000..7b3b6ac24d --- /dev/null +++ b/validation/Jenkinsfile.rancher-upgrade.e2e @@ -0,0 +1,1067 @@ +#!groovy +/** + * Dedicated Rancher server upgrade validation pipeline (rancher/tests#968, PRD #946). + * + * Six-stage flow, community leg: + * 1. Deploys a 2.14-latest environment via qa-infra-automation + * (make backend-s3, workspace-new, infra-up, cluster, rancher) + * 2. Pre-upgrade phase: installs Longhorn + seeds a checksum workload + * (validation/rancherupgrade, TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade) + * 3. Upgrades the Rancher server in place via `make rancher-upgrade` + * and re-injects the rotated admin token into the test config + * 4. Post-upgrade phase: asserts server version, Longhorn survival, workload data + * 5. Chart-upgrade phase: upgrades Longhorn to the version offered by the + * upgraded Rancher's catalog and re-verifies the workload data + * 6. Always destroys the environment (infra-down AUTO_APPROVE=yes) + * + * Versions resolve at runtime when RANCHER_VERSION_FROM / RANCHER_VERSION_TO_UPGRADE + * are empty (qa-infra-automation `make rancher-resolve-version`); pins override. + * + * Structurally cloned from validation/Jenkinsfile.e2e (deploy flow, admin-token + * injection, Qase testrun lifecycle, destroy post) and validation/pipeline/ + * Jenkinsfile.airgap-rke2-tests (make.runTarget call shapes, teardown guard). + * Unlike Jenkinsfile.e2e, this pipeline has no parallel downstream fan-out: the + * three test phases run sequentially against one shared Qase test run because + * each phase depends on the previous one's in-cluster state. + * + * Consumes shared functions from qa-jenkins-library: + * make.runTarget (backend-s3, workspace-new, infra-up, cluster, rancher, + * rancher-resolve-version, rancher-upgrade, infra-down) + * airgap.standardCheckout + * infrastructure.parseAndSubstituteVars, infrastructure.writeConfig, + * infrastructure.generateWorkspaceName, infrastructure.archiveWorkspaceName, + * infrastructure.writeSshKey, infrastructure.detectPublicIp + * property.useWithProperties + * + * Admin token generation uses a direct ansible-playbook call inside the + * rancher-go-test image (no make token target exists upstream yet). + */ + +def libraryBranch = env.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +library "qa-jenkins-library@${libraryBranch}" + +def DEFAULT_RANCHER_PASSWORD = 'X%S3bPyg@n8kWcRs675l#fH0' + +// Build-scoped image tags: every direct docker invocation below runs the +// pinned tag so this job never depends on (or mutates) whatever some other +// job last built under the shared :latest tag on the same agent. The :latest +// alias is still applied at build time because qa-jenkins-library's +// make.runTarget resolves that name for its own container runs. +def INFRA_TOOLS_IMAGE = "rancher-infra-tools:pitupgrade-${env.BUILD_NUMBER}" +def GO_TEST_IMAGE = "rancher-go-test:pitupgrade-${env.BUILD_NUMBER}" + +// Suite/method selectors passed as GOTEST_TESTCASE to the downstream executor. +// Each phase is its own go test invocation (-run regex), so cross-phase state +// lives in-cluster under deterministic names, never in process memory. +def PRE_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade"' +def POST_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade"' +def CHART_UPGRADE_TESTCASE = '-run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade"' + +/** + * Shared teardown helper: re-renders tfvars from params, tears down tofu workspace + * via make infra-down AUTO_APPROVE=yes. + * Sets env.INFRA_CLEANED to 'true' on success. + */ +def destroyInfrastructure(String reason) { + def wsName = '' + try { + if (fileExists('workspace_name.txt')) { + wsName = readFile('workspace_name.txt')?.trim() + } + } catch (e) { + echo "Could not read workspace_name.txt: ${e.message}" + } + + if (!wsName || !env.INFRA_DIR || env.INFRA_CLEANED == 'true') { + echo "Skipping teardown (${reason}): no workspace, no infra dir, or already cleaned" + return + } + + try { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + // Re-render terraform.tfvars so tofu destroy can access AWS state + def tofuDir = "${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes" + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME, + 'RUNNER_PUBLIC_IP': env.RUNNER_PUBLIC_IP + ] + ) + infrastructure.writeConfig( + path: "${tofuDir}/terraform.tfvars", + content: terraformConfig + ) + make.runTarget(target: 'infra-down', dir: env.INFRA_DIR, makeArgs: "WORKSPACE=${wsName} AUTO_APPROVE=yes") + env.INFRA_CLEANED = 'true' + } + } catch (cleanupErr) { + echo "Teardown failed (${reason}): ${cleanupErr.message}" + } +} + +/** + * Failure-only diagnostics: capture cluster state through the rancher-infra-tools + * container (kubectl + helm + kubeconfig from the deploy) into workspace files + * and archive them. Best-effort; never masks the failure it reports on. + */ +def collectUpgradeDiagnostics() { + if (!env.INFRA_DIR) { + echo 'Skipping diagnostics: no infra checkout available' + return + } + try { + def workspace = pwd() + sh """ + docker run --rm --platform linux/amd64 \ + -v ${workspace}:/workspace \ + -e KUBECONFIG=/workspace/${env.INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml \ + -w /workspace \ + ${INFRA_TOOLS_IMAGE} \ + sh -c ' + set +e + kubectl get nodes -o wide > diagnostics-nodes.txt 2>&1 + kubectl get pods -A > diagnostics-pods.txt 2>&1 + helm history rancher -n cattle-system > diagnostics-helm-history.txt 2>&1 + kubectl logs -n cattle-system -l app=rancher --tail=300 > diagnostics-rancher-logs.txt 2>&1 + kubectl logs -n cattle-cluster-system -l app=cattle-cluster-agent --tail=300 > diagnostics-agent-logs.txt 2>&1 + exit 0 + ' + """ + if (params.REPORT_ARTIFACTS) { + archiveArtifacts artifacts: 'diagnostics-*.txt', allowEmptyArchive: true + } + } catch (diagErr) { + echo "Diagnostics collection failed (non-fatal): ${diagErr.message}" + } +} + +pipeline { + agent any + + options { + ansiColor('xterm') + // Budget: three sequential test phases (TEST_TIMEOUT each, default 60m) + // plus deploy, server upgrade, and destroy (~90m overhead). Must exceed + // the sum of the stage budgets or a legitimately slow run aborts + // mid-phase; revisit when raising TEST_TIMEOUT. + timeout(time: 300, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '30')) + } + + parameters { + // ── Repository parameters ────────────────────────────── + string( + name: 'QA_JENKINS_LIBRARY_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-jenkins-library to use' + ) + // Repo params keep whatever the job was last triggered with: a + // declarative pipeline re-applies its parameters{} block to the job + // after each run, and hard-coded defaults here would clobber the + // JJB-configured per-controller values (fork + slice branches on the + // staging controller, upstream main on the main controller). + string( + name: 'TESTS_REPO_URL', + defaultValue: params.TESTS_REPO_URL ?: 'https://github.com/rancher/tests', + description: 'URL of rancher/tests repository' + ) + string( + name: 'TESTS_BRANCH', + defaultValue: params.TESTS_BRANCH ?: 'main', + description: 'Branch of rancher/tests repository' + ) + string( + name: 'QA_INFRA_REPO_URL', + defaultValue: params.QA_INFRA_REPO_URL ?: 'https://github.com/rancher/qa-infra-automation', + description: 'URL of qa-infra-automation repository' + ) + string( + name: 'QA_INFRA_BRANCH', + defaultValue: params.QA_INFRA_BRANCH ?: 'main', + description: 'Branch of qa-infra-automation repository' + ) + + // ── Upgrade parameters ───────────────────────────────── + string( + name: 'RANCHER_VERSION_FROM', + defaultValue: '', + description: 'Source Rancher version to deploy (empty = resolve latest 2.14.x patch at runtime)' + ) + string( + name: 'RANCHER_VERSION_TO_UPGRADE', + defaultValue: '', + description: 'Target Rancher version for the server upgrade (empty = resolve latest released final at runtime)' + ) + string( + name: 'RANCHER_CHART_REPO_FLAVOR', + defaultValue: 'community', + description: 'Chart repo flavor (community|prime): selects the chart index for version resolution, the repo the pre-upgrade Rancher is deployed from, and the repo the server upgrade pulls from' + ) + + // ── Infrastructure parameters ────────────────────────── + string( + name: 'HOSTNAME_PREFIX', + defaultValue: 'pitupgrade', + description: 'Hostname prefix for *.qa.rancher.space and other AWS resources' + ) + string( + name: 'RKE2_VERSION', + defaultValue: 'v1.35.1+rke2r1', + description: 'RKE2 version to deploy' + ) + password( + name: 'RANCHER_BOOTSTRAP_PASSWORD', + defaultValue: DEFAULT_RANCHER_PASSWORD, + description: 'Rancher bootstrap password for initial setup' + ) + password( + name: 'RANCHER_ADMIN_PASSWORD', + defaultValue: DEFAULT_RANCHER_PASSWORD, + description: 'Rancher admin password after bootstrap' + ) + booleanParam( + name: 'DESTROY_ON_FAILURE', + defaultValue: true, + description: 'Collect diagnostics and tear down infrastructure if pipeline fails' + ) + booleanParam( + name: 'DESTROY_AFTER_TESTS', + defaultValue: true, + description: 'Tear down infrastructure after a successful run; failed runs are governed by DESTROY_ON_FAILURE' + ) + + // ── Terraform parameters ─────────────────────────────── + string( + name: 'S3_BUCKET_NAME', + defaultValue: 'jenkins-terraform-state-storage', + description: 'S3 bucket name where Terraform state is stored' + ) + string( + name: 'S3_BUCKET_REGION', + defaultValue: 'us-east-2', + description: 'AWS region where the S3 bucket is located' + ) + string( + name: 'S3_KEY_PREFIX', + defaultValue: 'terraform.tfstate', + description: 'S3 key prefix for the Terraform state files' + ) + text( + name: 'TERRAFORM_CONFIG', + description: 'Terraform tfvars for the cluster_nodes module (non-airgap)', + defaultValue: '''aws_access_key = "${AWS_ACCESS_KEY_ID}" +aws_secret_key = "${AWS_SECRET_ACCESS_KEY}" +aws_ami = "ami-0600e22b69546c15c" +instance_type = "t3a.2xlarge" +aws_vpc = "vpc-bfccf4d7" +aws_subnet = "subnet-ee8cac86" +ephemeral_sg_ingress_cidrs = ["${RUNNER_PUBLIC_IP}/32"] +airgap_setup = false +proxy_setup = false +aws_volume_size = 500 +aws_volume_type = "gp3" +aws_hostname_prefix = "${HOSTNAME_PREFIX}" +aws_region = "us-east-2" +aws_route53_zone = "qa.rancher.space" +aws_ssh_user = "ubuntu" +public_ssh_key = "ssh_public_key.pub" +private_ssh_key = "~/.ssh/${AWS_SSH_PEM_KEY_NAME}" +nodes = [ + { + count = 3 + role = ["etcd", "cp", "worker"] + }, + { + count = 1 + role = ["worker"] + } +]''' + ) + text( + name: 'ANSIBLE_VARIABLES', + description: 'Ansible group_vars for RKE2 install + Rancher Helm deploy (non-airgap). Rancher version keys resolve from the pre-upgrade (FROM) version; rancher_version_upgrade keys hold the resolved target the server is upgraded to via make rancher-upgrade.', + defaultValue: '''--- +kubernetes_version: ${RKE2_VERSION} +cni: calico +kubeconfig_file: "/workspace/${INFRA_DIR}/ansible/rke2/default/kubeconfig.yaml" +cert_manager_version: "1.20.0" +bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +password: "${RANCHER_ADMIN_PASSWORD}" +rancher_version: ${RANCHER_VERSION} +rancher_image_tag: ${RANCHER_IMAGE_TAG} +rancher_chart_repo_url: "${RANCHER_CHART_REPO_URL}" +upgrade_mode: false +rancher_version_upgrade: "${RANCHER_VERSION_UPGRADE}" +rancher_image_tag_upgrade: "${RANCHER_IMAGE_TAG_UPGRADE}" +rancher_chart_repo_upgrade: "${RANCHER_UPGRADE_REPO_NAME}" +k8s_upgrade_mode: false +server_flags: | + ingress-controller: traefik''' + ) + + // ── Test parameters ──────────────────────────────────── + text( + name: 'CATTLE_TEST_CONFIG', + description: 'Test configuration YAML (cattle-config). rancher.adminToken is injected at runtime (initial + rotated after the server upgrade); rancherUpgradeInput.targetVersion is substituted from the resolved upgrade target.', + defaultValue: '''# The downstream per-phase executor can land on a different Jenkins agent +# than this orchestrator; awsNodeTemplate carries the three keys its +# Authorize Security Group stage needs (sg name, vpc, region) so the +# runner's public IP gets authorized on the tofu-created ephemeral SG +# (tf--sg) before the phases talk to the Rancher LB. +awsNodeTemplate: + region: us-east-2 + vpcId: vpc-bfccf4d7 + securityGroup: + - tf-${HOSTNAME_PREFIX}-sg +rancher: + host: ${HOSTNAME_PREFIX}.qa.rancher.space + adminPassword: ${ADMIN_PASSWORD} + insecure: true + # Deliberately false: pre-upgrade resources (Longhorn chart, PVC, workload) + # must survive into the post-upgrade phases; the pipeline environment + # teardown is the cleanup. + cleanup: false + # The qa-infra-automation cluster_nodes RKE2 cluster is Rancher's own + # ("local") cluster; the upgrade suite installs Longhorn on it. + clusterName: local +rancherUpgradeInput: + targetVersion: "${RANCHER_VERSION_UPGRADE}" +# Marker: set only by this pipeline. The suite fails rather than skips when +# this marker is present but the target version is missing (pipeline bug). +rancherUpgradePipeline: + enabled: true''', + ) + booleanParam( + name: 'REPORT_ARTIFACTS', + defaultValue: true, + description: 'Archive token metadata, Qase run config, and failure diagnostics as build artifacts' + ) + + // ── Downstream job parameters ───────────────────────── + string( + name: 'INDIVIDUAL_JOB', + defaultValue: params.INDIVIDUAL_JOB ?: 'go-pit-validation-job', + description: 'Name of the downstream Jenkins job that runs individual Go test suites' + ) + string( + name: 'TEST_PACKAGE', + defaultValue: './validation/rancherupgrade/...', + description: 'Go test package path passed to the downstream job' + ) + string( + name: 'TEST_TIMEOUT', + defaultValue: '60m', + description: 'Timeout for downstream test jobs' + ) + string( + name: 'QASE_REPORTER_SCRIPT', + defaultValue: 'build_qase_reporter_v2.sh', + description: 'Qase reporter script name in pipeline/scripts/' + ) + string( + name: 'QASE_SCHEMA_PREFIX', + defaultValue: 'pit', + description: 'Schema file prefix for Qase operations, for example hostbusters or pit' + ) + } + + environment { + WORKSPACE_NAME = '' + INFRA_CLEANED = 'false' + YQ_VERSION = 'v4.47.2' + // Folder-injected TEST_RUN_NAME (if present) wins; the fallback keeps + // the shared Qase run name readable when it is not injected. + TEST_RUN_NAME = "${env.TEST_RUN_NAME ?: 'Rancher Server Upgrade Validation'}" + } + + stages { + stage('Checkout') { + steps { + script { + def dirs = airgap.standardCheckout( + testsRepo: [url: params.TESTS_REPO_URL, branch: params.TESTS_BRANCH], + infraRepo: [url: params.QA_INFRA_REPO_URL, branch: params.QA_INFRA_BRANCH] + ) + env.TESTS_DIR = dirs.testsDir + env.INFRA_DIR = dirs.infraDir + + def hostnamePrefix = params.HOSTNAME_PREFIX ?: 'unknown' + currentBuild.displayName = "#${env.BUILD_NUMBER} - ${hostnamePrefix}" + currentBuild.description = "Rancher server upgrade validation for ${hostnamePrefix}" + } + } + } + + stage('Build Docker Images') { + steps { + // rancher-infra-tools — used by make.groovy / infra.* for all make + // targets (library resolves the :latest alias); direct invocations + // in this file use the build-pinned tag. + sh "docker build --no-cache --platform linux/amd64 -t ${INFRA_TOOLS_IMAGE} -t rancher-infra-tools:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.infra ." + // rancher-go-test — used for the qase binary build, token playbook, + // and redaction step, always via the build-pinned tag. + sh "docker build --no-cache --platform linux/amd64 -t ${GO_TEST_IMAGE} -t rancher-go-test:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.airgap-go-tests ." + } + } + + stage('Setup Tools') { + steps { + sh """ + wget -qO ./yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + wget -qO ./yq_checksums https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/checksums + grep -q '^yq_linux_amd64 ' ./yq_checksums + awk '/^yq_linux_amd64 / {print \$19 \" ./yq\"}' ./yq_checksums | sha256sum -c - + rm -f ./yq_checksums + chmod +x ./yq + """ + } + } + + // Auto-detect this runner's public IP so the ephemeral security + // group's SSH/LB ingress rule (ephemeral_sg_ingress_cidrs) allows + // the IP that Ansible/SSH will actually originate from. + stage('Detect Public IP') { + steps { + script { + env.RUNNER_PUBLIC_IP = infrastructure.detectPublicIp() + } + } + } + + // ── Resolve the upgrade window ───────────────────────── + // Empty params resolve at runtime via qa-infra-automation's + // rancher-resolve-version target; explicit pins pass through. + // The chosen lines/regexes and resolved versions are echoed so CI + // logs show exactly which version each stage ran against. + stage('Resolve Rancher Versions') { + steps { + script { + env.RANCHER_FROM_VERSION = params.RANCHER_VERSION_FROM?.trim() ?: '' + env.RANCHER_TO_VERSION = params.RANCHER_VERSION_TO_UPGRADE?.trim() ?: '' + + if (!env.RANCHER_FROM_VERSION) { + echo 'RANCHER_VERSION_FROM empty — resolving latest 2.14.x patch (SOURCE_LINE=2.14, latest final of the line)' + def resolveOut = make.runTarget( + target: 'rancher-resolve-version', + dir: env.INFRA_DIR, + makeArgs: [ + 'SOURCE_LINE=2.14', + "RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}" + ], + returnStdout: true, + passAwsCreds: false + ) + writeFile file: 'resolve-from.json', text: resolveOut + env.RANCHER_FROM_VERSION = sh( + script: "./yq '.chart_version' resolve-from.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + env.RANCHER_FROM_IMAGE_TAG = sh( + script: "./yq '.image_tag' resolve-from.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + } else { + env.RANCHER_FROM_IMAGE_TAG = env.RANCHER_FROM_VERSION.startsWith('v') ? env.RANCHER_FROM_VERSION : "v${env.RANCHER_FROM_VERSION}" + echo "RANCHER_VERSION_FROM pinned: ${env.RANCHER_FROM_VERSION} (image tag ${env.RANCHER_FROM_IMAGE_TAG})" + } + + if (!env.RANCHER_TO_VERSION) { + echo 'RANCHER_VERSION_TO_UPGRADE empty — resolving latest released final (no SOURCE_LINE, repo index latest final)' + def resolveOut = make.runTarget( + target: 'rancher-resolve-version', + dir: env.INFRA_DIR, + makeArgs: ["RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}"], + returnStdout: true, + passAwsCreds: false + ) + writeFile file: 'resolve-to.json', text: resolveOut + env.RANCHER_TO_VERSION = sh( + script: "./yq '.chart_version' resolve-to.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + env.RANCHER_TO_IMAGE_TAG = sh( + script: "./yq '.image_tag' resolve-to.json", + returnStdout: true + ).trim().replaceAll(/^"|"$/, '') + } else { + env.RANCHER_TO_IMAGE_TAG = env.RANCHER_TO_VERSION.startsWith('v') ? env.RANCHER_TO_VERSION : "v${env.RANCHER_TO_VERSION}" + echo "RANCHER_VERSION_TO_UPGRADE pinned: ${env.RANCHER_TO_VERSION} (image tag ${env.RANCHER_TO_IMAGE_TAG})" + } + + echo """ + Upgrade window resolved: + FROM (deploy): ${env.RANCHER_FROM_VERSION} (${env.RANCHER_FROM_IMAGE_TAG}) + TO (upgrade): ${env.RANCHER_TO_VERSION} (${env.RANCHER_TO_IMAGE_TAG}) + Chart flavor: ${params.RANCHER_CHART_REPO_FLAVOR} + """.stripIndent() + currentBuild.description = "Upgrade ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (${params.RANCHER_CHART_REPO_FLAVOR}) | ${params.HOSTNAME_PREFIX}" + } + } + } + + stage('Deploy Rancher Pre-Upgrade Environment') { + steps { + script { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/default" + def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/cluster_nodes" + def workspaceName = '' + + // ── SSH key setup ────────────────────────── + stage('Configure SSH Key') { + infrastructure.writeSshKey( + keyContent: env.AWS_SSH_PEM_KEY, + keyName: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ) + // Copy public key into the tofu module directory so + // tofu's file() function can read it (restricted to + // the module directory tree). + sh "cp .ssh/*.pub ${tofuModulePath}/ssh_public_key.pub" + } + + // ── Render terraform.tfvars ───────────────── + // Must exist before any make target runs (check-tofu-dir validates it). + stage('Configure Tofu Variables') { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME, + 'RUNNER_PUBLIC_IP': env.RUNNER_PUBLIC_IP + ] + ) + infrastructure.writeConfig( + path: "${tofuModulePath}/terraform.tfvars", + content: terraformConfig + ) + } + + // ── Initialize S3 backend ─────────────────── + stage('Initialize Tofu Backend') { + make.runTarget( + target: 'backend-s3', + dir: env.INFRA_DIR, + makeArgs: [ + "BUCKET=${params.S3_BUCKET_NAME}", + "KEY=${params.S3_KEY_PREFIX}", + "REGION=${params.S3_BUCKET_REGION}" + ] + ) + } + + // ── Create tofu workspace ─────────────────── + stage('Create Workspace') { + workspaceName = infrastructure.generateWorkspaceName( + prefix: 'jenkins_rancher_upgrade_workspace', + suffix: params.HOSTNAME_PREFIX, + includeTimestamp: false + ) + make.runTarget( + target: 'workspace-new', + dir: env.INFRA_DIR, + makeArgs: "WORKSPACE=${workspaceName}" + ) + infrastructure.archiveWorkspaceName(workspaceName: workspaceName) + env.WORKSPACE_NAME = workspaceName + } + + // ── Provision infra + generate inventory ──── + stage('Provision Infrastructure') { + make.runTarget( + target: 'infra-up', + dir: env.INFRA_DIR, + makeArgs: "WORKSPACE=${workspaceName}" + ) + } + + // ── Write group_vars/all.yml ──────────────── + // Must exist before make cluster (make infra-up generates + // inventory.yml but not the group_vars subdirectory). + // Rancher version keys carry the resolved FROM version; + // the *_upgrade keys carry the resolved TO version the + // later `make rancher-upgrade` stage targets. + stage('Configure Ansible Variables') { + sh "mkdir -p ${ansiblePath}/inventory/group_vars" + + // Flavor selects the deploy-side repo too: the prime + // leg installs the FROM version from the prime index + // (community stops the 2.14 line at 2.14.3 while prime + // continues it), keeping resolve, deploy, and upgrade + // all on the same artifact line. + def fromChartRepoUrl = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'https://charts.rancher.com/server-charts/prime' : 'https://releases.rancher.com/server-charts/latest' + def upgradeRepoName = params.RANCHER_CHART_REPO_FLAVOR == 'prime' ? 'rancher-prime-upgrade' : 'rancher-upgrade' + + def ansibleVars = infrastructure.parseAndSubstituteVars( + content: params.ANSIBLE_VARIABLES, + envVars: [ + 'RKE2_VERSION': params.RKE2_VERSION, + 'INFRA_DIR': env.INFRA_DIR, + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_VERSION': env.RANCHER_FROM_VERSION, + 'RANCHER_IMAGE_TAG': env.RANCHER_FROM_IMAGE_TAG, + 'RANCHER_CHART_REPO_URL': fromChartRepoUrl, + 'RANCHER_VERSION_UPGRADE': env.RANCHER_TO_VERSION, + 'RANCHER_IMAGE_TAG_UPGRADE': env.RANCHER_TO_IMAGE_TAG, + 'RANCHER_UPGRADE_REPO_NAME': upgradeRepoName + ] + ) + + def rancherVars = infrastructure.parseAndSubstituteVars( + content: '''deploy_rancher: true +install_helm: true +rancher_hostname: "${HOSTNAME_PREFIX}.qa.rancher.space" +rancher_bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +rancher_admin_password: "${RANCHER_ADMIN_PASSWORD}" +rancher_use_bundled_system_charts: true +''', + envVars: [ + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD + ] + ) + + // Merge ANSIBLE_VARIABLES + Rancher vars into a single group_vars/all.yml + infrastructure.writeConfig( + path: "${ansiblePath}/inventory/group_vars/all.yml", + content: ansibleVars + '\n' + rancherVars + ) + } + + // ── Deploy RKE2 cluster (ansible; no AWS creds) ── + stage('Deploy RKE2 Cluster') { + make.runTarget( + target: 'cluster', + dir: env.INFRA_DIR, + passAwsCreds: false + ) + } + + // ── Deploy pre-upgrade Rancher (helm; no AWS creds) ── + stage('Deploy Rancher') { + make.runTarget( + target: 'rancher', + dir: env.INFRA_DIR, + passAwsCreds: false + ) + } + + // ── Capture Rancher hostname ──────────────── + stage('Output Infrastructure Details') { + // Rancher hostname is deterministically constructed from HOSTNAME_PREFIX + // matching the value set in rancherVars (rancher_hostname). + def rancherHostname = "${params.HOSTNAME_PREFIX}.qa.rancher.space" + echo "Rancher Hostname: https://${rancherHostname}" + env.RANCHER_HOSTNAME = rancherHostname + } + } + } + } + } + + stage('Create Test Config and Inject Admin Token') { + steps { + script { + def workspace = pwd() + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + + // Create cattle-config with variable substitution, including + // the resolved upgrade target for rancherUpgradeInput.targetVersion. + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY' + ]) { + def resolvedConfig = infrastructure.parseAndSubstituteVars( + content: params.CATTLE_TEST_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: DEFAULT_RANCHER_PASSWORD, + 'RANCHER_VERSION_UPGRADE': env.RANCHER_TO_VERSION + ] + ) + writeFile file: cattleConfigPath, text: resolvedConfig.trim() + '\n' + } + + // Generate admin token via Ansible playbook + // TODO: replace with 'make token' once that target is added upstream + def containerConfigPath = "/workspace/${cattleConfigPath}" + def tokenTtl = env.RANCHER_TOKEN_TTL ?: '0' + def tokenDescription = "jenkins-rancher-upgrade-${env.BUILD_NUMBER}" + + // Diagnostic logging to validate env + output path before token generation + echo "Token generation context: workspace=${workspace}, config=${containerConfigPath}" + echo "RANCHER_ADMIN_PASSWORD present in params: ${params.RANCHER_ADMIN_PASSWORD ? 'yes' : 'no'}" + + sh """ + docker run --rm --platform linux/amd64 \ + --name generate-token \ + -e RANCHER_ADMIN_PASSWORD \ + -e RANCHER_URL=https://${params.HOSTNAME_PREFIX}.qa.rancher.space \ + -e ANSIBLE_CONFIG=/workspace/${env.INFRA_DIR}/ansible/ansible.cfg \ + -v ${workspace}:/workspace \ + -w /workspace/${env.INFRA_DIR}/ansible/rke2/default \ + ${GO_TEST_IMAGE} \ + ansible-playbook -i inventory/inventory.yml /workspace/${env.INFRA_DIR}/ansible/rancher/token/generate-admin-token.yml \ + -e rancher_cattle_config_file=${containerConfigPath} \ + -e rancher_token_ttl=${tokenTtl} \ + -e rancher_token_description=${tokenDescription} \ + -e rancher_token_output_format=json \ + -e rancher_token_output_file=/workspace/rancher-token.json \ + -e rancher_url=https://${params.HOSTNAME_PREFIX}.qa.rancher.space + """ + + def tokenFileExists = fileExists('rancher-token.json') + echo "Token file present after generation: ${tokenFileExists}" + + if (tokenFileExists) { + // Redact secret fields from token JSON via Docker container. + // Pipeline Utility Steps plugin (readJSON/writeJSON) is not installed, + // and python3 is not available on the Jenkins agent host. + sh "docker run --rm -v \$(pwd):/workspace -w /workspace ${GO_TEST_IMAGE} python3 -c \"import json; d=json.load(open('rancher-token.json')); [d.pop(k,None) for k in ('token','bearerToken','accessKey','secretKey','value')]; json.dump(d,open('rancher-token-metadata.json','w'),indent=2)\"" + if (params.REPORT_ARTIFACTS) { + archiveArtifacts artifacts: 'rancher-token-metadata.json' + echo 'Archived redacted token metadata: rancher-token-metadata.json' + } + + // Infrastructure, Rancher deploy, and admin token all succeeded. + // Shared library helpers (infra.*) may have set currentBuild.result to + // UNSTABLE/FAILURE via internal catchError() wrappers for non-fatal warnings, + // which would cause declarative pipeline to skip downstream test stages. + // Reset build result to SUCCESS so the upgrade phases can execute. + if (currentBuild.result && currentBuild.result != 'SUCCESS') { + echo "Resetting currentBuild.result from '${currentBuild.result}' to 'SUCCESS' — token generation succeeded, allowing upgrade phases to run" + currentBuild.result = 'SUCCESS' + } + } else { + error 'rancher-token.json not found after token generation — cannot run test phases without an admin token' + } + } + } + } + + // ── Phase 1: pre-upgrade Longhorn install + workload seed ── + // Starts the shared Qase run once; all three phases report into it. + stage('Pre-Upgrade Longhorn Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def workspace = pwd() + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + def testRunConfig = "${workspace}/testrunconfig.yaml" + def qaseBinary = "${workspace}/testrun" + + // Build qase testrun binary into the workspace root so the + // agent-side invocation below and testrunconfig.yaml (written + // to the binary's CWD) line up. + sh """ + docker run --rm --platform linux/amd64 \ + -v ${workspace}:/workspace \ + -w /workspace/${env.TESTS_DIR} \ + ${GO_TEST_IMAGE} \ + sh -c 'env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /workspace/testrun ./validation/pipeline/qase/testrun' + """ + + // Start the shared qase test run (completed after the final phase) + sh "${qaseBinary} --startRun=true" + + // Read test run ID + def testRunID = '' + if (fileExists(testRunConfig)) { + testRunID = sh( + script: "./yq '.id' ${testRunConfig}", + returnStdout: true + ).trim() + } + echo "Shared Qase test run started: id=${testRunID}" + if (params.REPORT_ARTIFACTS && fileExists(testRunConfig)) { + archiveArtifacts artifacts: 'testrunconfig.yaml', allowEmptyArchive: true + } + + // NOTE: unlike Jenkinsfile.e2e, rancher.cleanup is NOT set to + // true and no desiredflags are forced here — the upgrade suite's + // cross-phase state must survive, and the pipeline environment + // teardown is the cleanup. + def configYaml = readFile(file: cattleConfigPath) + + echo "Dispatching pre-upgrade phase: GOTEST_TESTCASE ${PRE_UPGRADE_TESTCASE}" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: PRE_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } + } + } + } + + // ── Phase 2: in-place Rancher server upgrade ── + // `make rancher-upgrade` rotates the Rancher admin token and writes the + // new one to ansible/rancher/default-ha/generated.tfvars; the old token + // is revoked, so the test config MUST be re-injected before the + // post-upgrade phases dispatch (verified live: old token 401s). + stage('Upgrade Rancher Server') { + steps { + script { + echo "Upgrading Rancher server: ${env.RANCHER_FROM_VERSION} -> ${env.RANCHER_TO_VERSION} (flavor ${params.RANCHER_CHART_REPO_FLAVOR})" + make.runTarget( + target: 'rancher-upgrade', + dir: env.INFRA_DIR, + makeArgs: [ + "ENV=default", + "RANCHER_VERSION_TO_UPGRADE=${env.RANCHER_TO_VERSION}", + "RANCHER_CHART_REPO_FLAVOR=${params.RANCHER_CHART_REPO_FLAVOR}" + ], + passAwsCreds: false + ) + + // Re-inject the rotated admin token into the test config + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + def generatedTfvars = "${env.INFRA_DIR}/ansible/rancher/default-ha/generated.tfvars" + + if (!fileExists(generatedTfvars)) { + error "Rotated token file not found at ${generatedTfvars} — post-upgrade phases cannot authenticate" + } + def rotatedToken = sh( + script: "awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}", + returnStdout: true + ).trim() + if (!rotatedToken) { + error "Could not extract api_key from ${generatedTfvars} — post-upgrade phases cannot authenticate" + } + echo "Rotated admin token extracted (length ${rotatedToken.length()}); re-injecting into test config" + + // The token never leaves this shell step (no Groovy/env + // interpolation, set +x keeps it out of the xtrace), + // and the injected token is smoke-checked against the + // live API before any phase is dispatched on it. + sh """ + set +x + token=\$(awk -F'\"' '/^api_key/{print \$2}' ${generatedTfvars}) + # export: yq's strenv() reads the environment, not the + # shell's unexported variables (empty-token 401 root cause). + export token + ./yq e '.rancher.adminToken = strenv(token)' -i ${cattleConfigPath} + status=\$(curl -sk -o /dev/null -w '%{http_code}' -H "Authorization: Bearer \$token" https://${params.HOSTNAME_PREFIX}.qa.rancher.space/v3/settings/server-version) + if [ "\$status" != "200" ]; then + echo "Rotated token rejected by the upgraded Rancher API: HTTP \$status (token length \${#token})" + exit 1 + fi + echo "Rotated token smoke-check: HTTP 200 from the upgraded Rancher API" + """ + echo "Test config now carries the post-upgrade admin token; rancherUpgradeInput.targetVersion=${env.RANCHER_TO_VERSION}" + } + } + } + + // ── Phase 3: post-upgrade assertions ── + stage('Post-Upgrade Longhorn Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def testRunID = sh( + script: "./yq '.id' testrunconfig.yaml", + returnStdout: true + ).trim() + def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + + // Fail fast (without printing the token) if the config + // about to be dispatched does not carry the rotated one. + sh """ + set +x + expected=\$(awk -F'\"' '/^api_key/{print \$2}' ${env.INFRA_DIR}/ansible/rancher/default-ha/generated.tfvars) + actual=\$(./yq '.rancher.adminToken' ${env.TESTS_DIR}/cattle-config.yaml) + if [ "\$expected" != "\$actual" ]; then + echo "Dispatched config adminToken does not match the rotated token (expected length \${#expected}, got length \${#actual})" + exit 1 + fi + echo "Dispatched config carries the rotated admin token (length \${#expected})" + """ + + echo "Dispatching post-upgrade phase: GOTEST_TESTCASE ${POST_UPGRADE_TESTCASE} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: POST_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + } + } + } + } + + // ── Phase 4: Longhorn chart upgrade through the upgraded catalog ── + // Completes the shared Qase run after this final phase; do not rely on + // the downstream job's own QASE_TEST_RUN_COMPLETE for the shared run. + stage('Longhorn Chart Upgrade Tests') { + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + def testRunID = sh( + script: "./yq '.id' testrunconfig.yaml", + returnStdout: true + ).trim() + def configYaml = readFile(file: "${env.TESTS_DIR}/cattle-config.yaml") + + echo "Dispatching chart-upgrade phase: GOTEST_TESTCASE ${CHART_UPGRADE_TESTCASE} (Qase run ${testRunID})" + build job: params.INDIVIDUAL_JOB, parameters: [ + string(name: 'TIMEOUT', value: params.TEST_TIMEOUT), + text(name: 'CONFIG', value: configYaml), + string(name: 'REPO', value: params.TESTS_REPO_URL), + string(name: 'BRANCH', value: params.TESTS_BRANCH), + string(name: 'QASE_TEST_RUN_ID', value: testRunID), + string(name: 'QASE_SCHEMA_PREFIX', value: params.QASE_SCHEMA_PREFIX), + string(name: 'QASE_REPORTER_SCRIPT', value: params.QASE_REPORTER_SCRIPT), + string(name: 'TEST_PACKAGE', value: params.TEST_PACKAGE), + string(name: 'GOTEST_TESTCASE', value: CHART_UPGRADE_TESTCASE), + string(name: 'TAGS', value: 'validation'), + booleanParam(name: 'CAPTURE_IMAGES', value: false) + ] + + // Complete the shared qase test run + sh "./testrun --startRun=false" + env.QASE_RUN_COMPLETED = 'true' + echo "Shared Qase test run ${testRunID} completed" + } + } + } + } + } + + post { + always { + script { + // Failure diagnostics run first, while the live cluster and + // the docker images still exist: Jenkins evaluates post + // conditions in a fixed order (always before failure, cleanup + // last), so every failure-path action must run in this block, + // ahead of the teardown below. + if (currentBuild.currentResult != 'SUCCESS') { + collectUpgradeDiagnostics() + } + + // Complete the shared Qase run when a phase failed before the + // final stage could close it (no-op once already completed). + if (env.QASE_RUN_COMPLETED != 'true' && fileExists('testrun') && fileExists('testrunconfig.yaml')) { + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + sh './testrun --startRun=false' + } + env.QASE_RUN_COMPLETED = 'true' + } + } + + // Single teardown decision point. The failure-path destroy is + // decided here rather than in a failure{} block because always + // runs first: destroyInfrastructure needs workspace_name.txt, + // the tfvars and the infra-tools image, which are only removed + // afterwards in cleanup{}. A failed run is governed by + // DESTROY_ON_FAILURE alone and a successful one by + // DESTROY_AFTER_TESTS, so DESTROY_ON_FAILURE=false on a failed + // run keeps the environment around for debugging even when + // DESTROY_AFTER_TESTS is true. + def buildSucceeded = currentBuild.currentResult == 'SUCCESS' + def shouldDestroy = buildSucceeded ? params.DESTROY_AFTER_TESTS : params.DESTROY_ON_FAILURE + if (shouldDestroy) { + destroyInfrastructure(buildSucceeded ? 'DESTROY_AFTER_TESTS' : 'DESTROY_ON_FAILURE') + } + + } + } + + cleanup { + script { + // Docker cleanup — cleanup{} runs after always, so the + // teardown above can still resolve the infra-tools image. + sh "docker rmi -f ${GO_TEST_IMAGE} ${INFRA_TOOLS_IMAGE} rancher-go-test:latest rancher-infra-tools:latest || true" + + // Sensitive artifact cleanup — shred or remove credentials from workspace + sh ''' + set +e + + cleanup_file() { + if [ -f "$1" ]; then + if command -v shred >/dev/null 2>&1; then + shred -u "$1" || rm -f "$1" + else + rm -f "$1" + fi + fi + } + + cleanup_dir() { + if [ -d "$1" ]; then + find "$1" -type f -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + rm -rf "$1" + fi + } + + cleanup_file ".go_test_env" + cleanup_file "rancher-token.json" + cleanup_file "rancher-token-metadata.json" + cleanup_file "workspace_name.txt" + cleanup_file "resolve-from.json" + cleanup_file "resolve-to.json" + # cattle-config carries the admin password and the live rotated + # admin token; group_vars/all.yml carries the rancher passwords. + cleanup_file "${TESTS_DIR}/cattle-config.yaml" + + if [ -n "${INFRA_DIR}" ] && [ -d "${INFRA_DIR}" ]; then + find "${INFRA_DIR}" -type f \\( -name 'terraform.tfvars' -o -name 'generated.tfvars' -o -name 'all.yml' -o -name '*.pem' -o -name 'id_rsa' -o -name 'id_ed25519' \\) -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + fi + + cleanup_dir ".ssh" + ''' + } + } + } +} diff --git a/validation/charts/neuvector.go b/validation/charts/neuvector.go index 0a16019bf8..47027ed55e 100644 --- a/validation/charts/neuvector.go +++ b/validation/charts/neuvector.go @@ -26,9 +26,9 @@ var expectedNeuVectorServices = []string{ neuVectorAdmissionWebhookSvc, } -// verifyNeuVectorServicesActive checks that all expected NeuVector services exist and are active +// VerifyNeuVectorServicesActive checks that all expected NeuVector services exist and are active // in the NeuVector namespace. -func verifyNeuVectorServicesActive(client *rancher.Client, clusterID string) error { +func VerifyNeuVectorServicesActive(client *rancher.Client, clusterID string) error { steveClient, err := client.Steve.ProxyDownstream(clusterID) if err != nil { return err @@ -51,9 +51,9 @@ func verifyNeuVectorServicesActive(client *rancher.Client, clusterID string) err return nil } -// verifyNeuVectorWebUIAccessible verifies that the NeuVector manager web UI is reachable +// VerifyNeuVectorWebUIAccessible verifies that the NeuVector manager web UI is reachable // through the Rancher service proxy endpoint. -func verifyNeuVectorWebUIAccessible(client *rancher.Client, clusterID string) (string, error) { +func VerifyNeuVectorWebUIAccessible(client *rancher.Client, clusterID string) (string, error) { proxyPath := fmt.Sprintf(neuVectorWebUIServiceProxyFmt, clusterID, charts.NeuVectorNamespace, neuVectorWebUIService, neuVectorWebUIPort) logrus.Infof("Verifying NeuVector manager UI is accessible at path [%s]", proxyPath) diff --git a/validation/charts/neuvector_test.go b/validation/charts/neuvector_test.go index 140577a612..5822ac87e9 100644 --- a/validation/charts/neuvector_test.go +++ b/validation/charts/neuvector_test.go @@ -108,7 +108,7 @@ func (n *NeuVectorTestSuite) TestNeuVectorInstallation() { require.NoError(n.T(), err) n.T().Log("Verifying all expected NeuVector services are active") - err = verifyNeuVectorServicesActive(client, n.cluster.ID) + err = VerifyNeuVectorServicesActive(client, n.cluster.ID) require.NoError(n.T(), err) if n.chartInstallOptions.DefaultRegistry != "" { @@ -119,7 +119,7 @@ func (n *NeuVectorTestSuite) TestNeuVectorInstallation() { } n.T().Log("Verifying NeuVector manager web UI is accessible via service proxy") - respUI, err := verifyNeuVectorWebUIAccessible(client, n.cluster.ID) + respUI, err := VerifyNeuVectorWebUIAccessible(client, n.cluster.ID) require.NoError(n.T(), err, "NeuVector manager web UI should be accessible via Rancher service proxy") require.NotEmpty(n.T(), respUI, "Expected non-empty response from NeuVector manager web UI") } diff --git a/validation/rancherupgrade/README.md b/validation/rancherupgrade/README.md new file mode 100644 index 0000000000..901b0f5b32 --- /dev/null +++ b/validation/rancherupgrade/README.md @@ -0,0 +1,101 @@ +# Rancher Upgrade + +The `rancherupgrade` package validates upgrading the Rancher server from the previous +release line to the latest (currently 2.14.x -> 2.15.x) with interoperability products +installed. The line pair is the pipeline's default, not a suite constraint: the suite +itself is version-agnostic and gates on `rancherUpgradeInput.targetVersion`, while the +pipeline resolves the source via `RANCHER_VERSION_FROM` (empty = latest 2.14.x patch via +`SOURCE_LINE=2.14`) and the target via `RANCHER_VERSION_TO_UPGRADE` (empty = latest +released final). The first vertical is Longhorn: a pre-upgrade +phase installs the Longhorn chart and seeds a checksum workload on a Longhorn volume, +the Rancher server is then upgraded out-of-band (qa-infra-automation `make rancher-upgrade`), +and post-upgrade phases verify the server version, the surviving Longhorn installation, +the workload data, and a Longhorn chart upgrade to the version offered by the upgraded +Rancher's catalog. + +Each phase is a suite test method selected via the `-run` regex and runs as its own +`go test` invocation in its own pipeline stage. All cross-phase state lives in-cluster +under deterministic names (PVC/deployment `rancher-upgrade-longhorn`, data file +`/auto-mnt/checksum-data`), never in process memory. + +## Deliberate deviations + +Do not "fix" the following; both are required by the upgrade workflow (rancher/tests#967): + +- `TearDownSuite` does not call `session.Cleanup()` and the helpers register no cleanup + funcs: pre-phase resources (Longhorn chart, PVC, workload) must survive into the + later phases, which run as separate processes. The pipeline environment teardown is + the cleanup. +- The suite is admin-only: upgrade phases assert server-level state and the pipeline + runs them with the admin token. +- The server upgrade stays out-of-band (a pipeline stage running qa-infra-automation + `make rancher-upgrade`) rather than a step inside the test: the go test phases + assert outcomes only and hold no infra credentials, the upgrade needs the + controller-side qa-infra checkout and its terraform/ansible credentials, and + per-phase Jenkins stage status is the acceptance signal; pulling the upgrade into + the test process would collapse the three stages into one. +- The build tag is `(validation || pit.weekly)`: generic scheduled PIT legs deploy + Rancher once and never upgrade it, so `SetupSuite` skips the suite when + `rancherUpgradeInput.targetVersion` is unset. The dedicated pipeline marks its + config with `rancherUpgradePipeline.enabled: true`; with the marker present but + the target missing, the suite fails (pipeline bug) instead of skipping. + +## Config + +```yaml +rancher: + host: + adminToken: + insecure: true + clusterName: "local" # Rancher's own local cluster: the qa-infra-automation RKE2 HA deploy + +rancherUpgradeInput: + targetVersion: "" +``` +`rancher.clusterName` selects the target cluster. The pipeline uses `local` — Rancher's own +3-node HA RKE2 cluster provisioned by qa-infra-automation — so Longhorn and its volume are +exercised on the server cluster itself, matching what the pipeline deploys. +`rancherUpgradeInput.targetVersion` is required by every phase (the suite skips without it); +the pipeline writes it from the qa-infra-automation resolver output, or set it by hand for +local runs. + +## Running + +Pre-upgrade (installs Longhorn at the 2.14-era latest and seeds the checksum workload; +run against the not-yet-upgraded Rancher): + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornPreUpgrade" +``` + +Post-upgrade (after `make rancher-upgrade RANCHER_VERSION_TO_UPGRADE= RANCHER_CHART_REPO_FLAVOR=community|prime` +in qa-infra-automation, and with `rancherUpgradeInput.targetVersion` set): + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornPostUpgrade" +``` + +Chart upgrade: + +```bash +gotestsum --format standard-verbose --packages=github.com/rancher/tests/validation/rancherupgrade --junitfile results.xml -- -timeout=60m -tags="validation" -v -run "TestLonghornUpgradeTestSuite/TestLonghornChartUpgrade" +``` + +## Notes + +- The target cluster is Rancher's own local cluster (the qa-infra-automation 3-node HA + RKE2 deploy), so the server nodes themselves must be sized for Longhorn plus the later + NeuVector/Observability suites; this is an infra concern and is not configurable here. +- The Rancher server upgrade itself is `make rancher-upgrade` in + [qa-infra-automation](https://github.com/rancher/qa-infra-automation); this package + only runs the pre- and post-upgrade assertions around it. +- `make rancher-upgrade` rotates the admin token and writes the new one to + `ansible/rancher/-ha/generated.tfvars`; the pipeline must copy it into the test + config's `rancher.adminToken` before the post-upgrade phases (the pre-upgrade token + is revoked by the upgrade). +- After a server upgrade the cluster-scoped catalog follower can keep serving the + previous Rancher line's index (e.g. release-v2.14 after upgrading to 2.15) until the + `cattle-cluster-agent` re-syncs. The chart-upgrade phase detects this and fails with + an explicit stale-catalog message when the catalog's latest equals the installed + version, or with "failed to find chart longhorn from the chart repo" on a lookup + miss; restart the cluster agent and re-run the phase. Observed re-sync time ~4 minutes. diff --git a/validation/rancherupgrade/longhorn_test.go b/validation/rancherupgrade/longhorn_test.go new file mode 100644 index 0000000000..3b203420d6 --- /dev/null +++ b/validation/rancherupgrade/longhorn_test.go @@ -0,0 +1,182 @@ +//go:build (validation || pit.weekly) && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 + +package rancherupgrade + +import ( + "testing" + + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + "github.com/rancher/tests/actions/charts" + actionsClusters "github.com/rancher/tests/actions/clusters" + "github.com/rancher/tests/actions/projects" + "github.com/rancher/tests/actions/storage" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type LonghornUpgradeTestSuite struct { + suite.Suite + session *session.Session + client *rancher.Client + cluster *clusters.ClusterMeta +} + +const ( + // checksumWorkloadName is the deterministic name shared by the PVC and the nginx + // deployment created in the pre-upgrade phase and relocated by later phases. + checksumWorkloadName = "rancher-upgrade-longhorn" + + // checksumContent is the shell-safe data written to the checksum file on the + // Longhorn-backed volume before the Rancher upgrade and verified after it. + checksumContent = "rancher-upgrade-longhorn-checksum-v1" +) + +// loadUpgradeInput returns the rancherUpgradeInput section of the test config. +func loadUpgradeInput() *UpgradeInput { + upgradeInput := new(UpgradeInput) + config.LoadConfig(UpgradeInputConfigKey, upgradeInput) + + return upgradeInput +} + +func (s *LonghornUpgradeTestSuite) SetupSuite() { + cfg := loadUpgradeInput() + pipeline := new(PipelineInput) + config.LoadConfig(PipelineConfigKey, pipeline) + + if cfg.TargetVersion == "" { + // The dedicated pipeline always writes both keys, so an empty target + // there is a pipeline bug and must fail, not skip. + require.False(s.T(), pipeline.Enabled, "rancher-upgrade pipeline config is missing %s.targetVersion", UpgradeInputConfigKey) + // Generic PIT legs never upgrade the server between phases; without + // an upgrade target the phases have nothing to assert, so skip. + s.T().Skipf("%s.targetVersion is not set: this suite runs only in the rancher-upgrade pipeline, which upgrades the server between phases", UpgradeInputConfigKey) + } + + testSession := session.NewSession() + s.session = testSession + + client, err := rancher.NewClient("", testSession) + require.NoError(s.T(), err) + s.client = client + + s.cluster, err = clusters.NewClusterMeta(client, client.RancherConfig.ClusterName) + require.NoError(s.T(), err) +} + +// TearDownSuite deliberately does not call session.Cleanup(): resources created in +// the pre-upgrade phase (Longhorn chart, PVC, workload) must survive into the +// post-upgrade and chart-upgrade phases, which run as separate go test processes. +// The pipeline environment teardown is the cleanup for these resources. +func (s *LonghornUpgradeTestSuite) TearDownSuite() { +} + +func (s *LonghornUpgradeTestSuite) TestLonghornPreUpgrade() { + logrus.Infof("pre-upgrade phase: creating project on cluster %s", s.cluster.ID) + project, _, err := projects.CreateProjectAndNamespace(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + // Resolve the version from the cluster-scoped catalog and install that: the + // management view can offer versions the cluster-scoped shard does not serve + // yet (rancher-2.15-catalog-sharding-issue.md), and the chart action validates + // against the cluster-scoped index. The pre-phase still yields the 2.14-era + // version by construction, since it runs against the pre-upgrade Rancher. + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + version, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: installing longhorn %s from the %s repo", version, catalog.RancherChartRepo) + + err = charts.InstallLonghornChart(s.client, charts.PayloadOpts{ + Namespace: charts.LonghornNamespace, + Host: s.client.RancherConfig.Host, + InstallOptions: charts.InstallOptions{ + Cluster: s.cluster, + Version: version, + ProjectID: project.ID, + }, + }, nil) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: longhorn %s installed; seeding checksum workload %s", version, checksumWorkloadName) + + err = storage.CreateChecksumWorkload(s.client, s.cluster.ID, charts.LonghornStorageClass, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: checksum workload %s ready; pre-upgrade state seeded", checksumWorkloadName) +} + +func (s *LonghornUpgradeTestSuite) TestLonghornPostUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("post-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: server version %s observed; checking longhorn survival", cfg.TargetVersion) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.True(s.T(), chart.IsAlreadyInstalled, "longhorn chart must still be installed after the Rancher server upgrade") + logrus.Infof("post-upgrade phase: longhorn chart still installed; verifying workload data") + + err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: checksum verified on %s", checksumWorkloadName) +} + +func (s *LonghornUpgradeTestSuite) TestLonghornChartUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("chart-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + + // Resolve the latest version from the cluster-scoped catalog: the management + // view can report versions absent from the cluster-scoped index (see + // rancher-2.15-catalog-sharding-issue.md at the repo root). + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + latest, err := catalogClient.GetLatestChartVersion(charts.LonghornChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + // Guard against a stale cluster-scoped catalog: until the cluster agent + // re-syncs after the server upgrade, the follower can still serve the + // previous Rancher line's index, whose latest equals the installed version. + // Upgrading to it would pass vacuously (a same-version upgrade asserted + // equal to itself), so fail with the runbook remedy instead; see README Notes. + installed, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.NotEqual(s.T(), + installed.ChartDetails.Spec.Chart.Metadata.Version, latest, + "cluster-scoped catalog still serves the pre-upgrade index: installed longhorn %s equals latest %s; restart the cluster agent and re-run this phase", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + logrus.Infof("chart-upgrade phase: upgrading longhorn %s to %s from the upgraded catalog", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + + installOptions := &charts.InstallOptions{ + Cluster: s.cluster, + Version: latest, + } + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return charts.UpgradeLonghornChart(s.client, installOptions) + }) + require.NoError(s.T(), err) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.LonghornNamespace, charts.LonghornChartName) + require.NoError(s.T(), err) + require.Equal(s.T(), latest, chart.ChartDetails.Spec.Chart.Metadata.Version) + logrus.Infof("chart-upgrade phase: longhorn at %s; verifying workload data", latest) + + err = storage.VerifyChecksumWorkload(s.client, s.cluster.ID, checksumWorkloadName, checksumContent) + require.NoError(s.T(), err) + logrus.Infof("chart-upgrade phase: checksum verified on %s", checksumWorkloadName) +} + +func TestLonghornUpgradeTestSuite(t *testing.T) { + suite.Run(t, new(LonghornUpgradeTestSuite)) +} diff --git a/validation/rancherupgrade/neuvector_upgrade_test.go b/validation/rancherupgrade/neuvector_upgrade_test.go new file mode 100644 index 0000000000..bea6d59931 --- /dev/null +++ b/validation/rancherupgrade/neuvector_upgrade_test.go @@ -0,0 +1,235 @@ +//go:build (validation || pit.weekly) && !2.8 && !2.9 && !2.10 && !2.11 && !2.12 && !2.13 + +package rancherupgrade + +import ( + "fmt" + "testing" + + "github.com/rancher/shepherd/clients/rancher" + "github.com/rancher/shepherd/clients/rancher/catalog" + management "github.com/rancher/shepherd/clients/rancher/generated/management/v3" + shepherdCharts "github.com/rancher/shepherd/extensions/charts" + "github.com/rancher/shepherd/extensions/clusters" + "github.com/rancher/shepherd/extensions/defaults" + "github.com/rancher/shepherd/pkg/config" + "github.com/rancher/shepherd/pkg/session" + "github.com/rancher/tests/actions/charts" + actionsClusters "github.com/rancher/tests/actions/clusters" + "github.com/rancher/tests/actions/neuvector" + "github.com/rancher/tests/actions/projects" + validationCharts "github.com/rancher/tests/validation/charts" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type NeuVectorUpgradeTestSuite struct { + suite.Suite + session *session.Session + client *rancher.Client + cluster *clusters.ClusterMeta +} + +// nvAdmissionDenyNamespace is the deterministic namespace protected by the +// NeuVector admission control deny rule: workload creates in it must be +// rejected by the admission webhook before and after the Rancher server +// upgrade and again after the NeuVector chart upgrade. The deny namespace and +// the NvAdmissionControlSecurityRule are created in the pre-upgrade phase and +// must survive every later phase, which run as separate go test processes. +const nvAdmissionDenyNamespace = "rancher-upgrade-neuvector-deny" + +// nvSystemProject returns the System project of the target cluster, which the +// NeuVector chart is installed into, the same placement the hardened NeuVector +// suite uses. +func nvSystemProject(client *rancher.Client, clusterID string) (*management.Project, error) { + project, err := projects.GetProjectByName(client, clusterID, charts.SystemProject) + if err != nil { + return nil, err + } + + if project.Name != charts.SystemProject { + return nil, fmt.Errorf("expected project %q on cluster %s, got %q", charts.SystemProject, clusterID, project.Name) + } + + return project, nil +} + +// nvSystemDefaultRegistry returns the system-default-registry setting value. It +// is empty outside airgap environments and must be passed to the NeuVector chart +// install so the images come from the mirrored registry in airgap environments. +func nvSystemDefaultRegistry(client *rancher.Client) (string, error) { + registrySetting, err := client.Management.Setting.ByID("system-default-registry") + if err != nil { + return "", err + } + + return registrySetting.Value, nil +} + +func (s *NeuVectorUpgradeTestSuite) SetupSuite() { + cfg := loadUpgradeInput() + pipeline := new(PipelineInput) + config.LoadConfig(PipelineConfigKey, pipeline) + + if cfg.TargetVersion == "" { + // The dedicated pipeline always writes both keys, so an empty target + // there is a pipeline bug and must fail, not skip. + require.False(s.T(), pipeline.Enabled, "rancher-upgrade pipeline config is missing %s.targetVersion", UpgradeInputConfigKey) + // Generic PIT legs never upgrade the server between phases; without + // an upgrade target the phases have nothing to assert, so skip. + s.T().Skipf("%s.targetVersion is not set: this suite runs only in the rancher-upgrade pipeline, which upgrades the server between phases", UpgradeInputConfigKey) + } + + testSession := session.NewSession() + s.session = testSession + + client, err := rancher.NewClient("", testSession) + require.NoError(s.T(), err) + s.client = client + + s.cluster, err = clusters.NewClusterMeta(client, client.RancherConfig.ClusterName) + require.NoError(s.T(), err) +} + +// TearDownSuite deliberately does not call session.Cleanup(): resources created in +// the pre-upgrade phase (NeuVector chart, admission control deny rule and its deny +// namespace) must survive into the post-upgrade and chart-upgrade phases, which +// run as separate go test processes. The pipeline environment teardown is the +// cleanup for these resources. +func (s *NeuVectorUpgradeTestSuite) TearDownSuite() { +} + +func (s *NeuVectorUpgradeTestSuite) TestNeuVectorPreUpgrade() { + logrus.Infof("pre-upgrade phase: resolving the System project on cluster %s", s.cluster.ID) + project, err := nvSystemProject(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + // Resolve the version from the cluster-scoped catalog and install that: the + // management view can offer versions the cluster-scoped shard does not serve + // yet (rancher-2.15-catalog-sharding-issue.md), and the chart action validates + // against the cluster-scoped index. The pre-phase still yields the 2.14-era + // version by construction, since it runs against the pre-upgrade Rancher. + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + version, err := catalogClient.GetLatestChartVersion(charts.NeuVectorChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + defaultRegistry, err := nvSystemDefaultRegistry(s.client) + require.NoError(s.T(), err) + + logrus.Infof("pre-upgrade phase: installing neuvector %s from the %s repo", version, catalog.RancherChartRepo) + err = charts.InstallNeuVectorChart(s.client, charts.PayloadOpts{ + Namespace: charts.NeuVectorNamespace, + Host: s.client.RancherConfig.Host, + InstallOptions: charts.InstallOptions{ + Cluster: s.cluster, + Version: version, + ProjectID: project.ID, + }, + DefaultRegistry: defaultRegistry, + K3s: s.cluster.Provider == clusters.KubernetesProviderK3S, + }) + require.NoError(s.T(), err) + + logrus.Infof("pre-upgrade phase: neuvector %s installed; waiting for its workloads", version) + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return shepherdCharts.WaitChartInstall(catalogClient, charts.NeuVectorNamespace, charts.NeuVectorChartName) + }) + require.NoError(s.T(), err) + + err = shepherdCharts.WatchAndWaitDeployments(s.client, s.cluster.ID, charts.NeuVectorNamespace, metav1.ListOptions{}) + require.NoError(s.T(), err) + err = shepherdCharts.WatchAndWaitDaemonSets(s.client, s.cluster.ID, charts.NeuVectorNamespace, metav1.ListOptions{}) + require.NoError(s.T(), err) + + logrus.Infof("pre-upgrade phase: enabling the admission control deny rule for namespace %s", nvAdmissionDenyNamespace) + err = neuvector.EnableAdmissionControlDenyNamespace(s.client, s.cluster.ID, nvAdmissionDenyNamespace) + require.NoError(s.T(), err) + + err = neuvector.VerifyAdmissionDenial(s.client, s.cluster.ID, nvAdmissionDenyNamespace) + require.NoError(s.T(), err) + logrus.Infof("pre-upgrade phase: admission denial proven in %s; pre-upgrade state seeded", nvAdmissionDenyNamespace) +} + +func (s *NeuVectorUpgradeTestSuite) TestNeuVectorPostUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("post-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: server version %s observed; checking neuvector survival", cfg.TargetVersion) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.NeuVectorNamespace, charts.NeuVectorChartName) + require.NoError(s.T(), err) + require.True(s.T(), chart.IsAlreadyInstalled, "neuvector chart must still be installed after the Rancher server upgrade") + logrus.Infof("post-upgrade phase: neuvector chart still installed; verifying services and manager UI") + + err = validationCharts.VerifyNeuVectorServicesActive(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + respUI, err := validationCharts.VerifyNeuVectorWebUIAccessible(s.client, s.cluster.ID) + require.NoError(s.T(), err, "neuvector manager web UI should be reachable via the Rancher service proxy") + require.NotEmpty(s.T(), respUI, "expected a non-empty response from the neuvector manager web UI") + logrus.Infof("post-upgrade phase: services active and manager UI reachable; verifying admission enforcement") + + err = neuvector.VerifyAdmissionDenial(s.client, s.cluster.ID, nvAdmissionDenyNamespace) + require.NoError(s.T(), err) + logrus.Infof("post-upgrade phase: admission webhook still denies creates in %s", nvAdmissionDenyNamespace) +} + +func (s *NeuVectorUpgradeTestSuite) TestNeuVectorChartUpgrade() { + cfg := loadUpgradeInput() + logrus.Infof("chart-upgrade phase: waiting for rancher server version %s", cfg.TargetVersion) + + err := actionsClusters.WaitRancherVersion(s.client, cfg.TargetVersion, defaults.TenMinuteTimeout) + require.NoError(s.T(), err) + + // Resolve the latest version from the cluster-scoped catalog: the management + // view can report versions absent from the cluster-scoped index (see + // rancher-2.15-catalog-sharding-issue.md at the repo root). + catalogClient, err := s.client.GetClusterCatalogClient(s.cluster.ID) + require.NoError(s.T(), err) + latest, err := catalogClient.GetLatestChartVersion(charts.NeuVectorChartName, catalog.RancherChartRepo) + require.NoError(s.T(), err) + + // Guard against a stale cluster-scoped catalog: until the cluster agent + // re-syncs after the server upgrade, the follower can still serve the + // previous Rancher line's index, whose latest equals the installed version. + // Upgrading to it would pass vacuously (a same-version upgrade asserted + // equal to itself), so fail with the runbook remedy instead; see README Notes. + installed, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.NeuVectorNamespace, charts.NeuVectorChartName) + require.NoError(s.T(), err) + require.NotEqual(s.T(), + installed.ChartDetails.Spec.Chart.Metadata.Version, latest, + "cluster-scoped catalog still serves the pre-upgrade index: installed neuvector %s equals latest %s; restart the cluster agent and re-run this phase", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + logrus.Infof("chart-upgrade phase: upgrading neuvector %s to %s from the upgraded catalog", + installed.ChartDetails.Spec.Chart.Metadata.Version, latest) + + installOptions := &charts.InstallOptions{ + Cluster: s.cluster, + Version: latest, + } + err = charts.RetryOnWatchError(charts.DefaultWatchRetries, func() error { + return charts.UpgradeNeuVectorChart(s.client, installOptions) + }) + require.NoError(s.T(), err) + + chart, err := shepherdCharts.GetChartStatus(s.client, s.cluster.ID, charts.NeuVectorNamespace, charts.NeuVectorChartName) + require.NoError(s.T(), err) + require.Equal(s.T(), latest, chart.ChartDetails.Spec.Chart.Metadata.Version) + logrus.Infof("chart-upgrade phase: neuvector at %s; verifying health and enforcement", latest) + + err = validationCharts.VerifyNeuVectorServicesActive(s.client, s.cluster.ID) + require.NoError(s.T(), err) + + err = neuvector.VerifyAdmissionDenial(s.client, s.cluster.ID, nvAdmissionDenyNamespace) + require.NoError(s.T(), err) + logrus.Infof("chart-upgrade phase: admission webhook still denies creates in %s", nvAdmissionDenyNamespace) +} + +func TestNeuVectorUpgradeTestSuite(t *testing.T) { + suite.Run(t, new(NeuVectorUpgradeTestSuite)) +} diff --git a/validation/rancherupgrade/rancherupgrade.go b/validation/rancherupgrade/rancherupgrade.go new file mode 100644 index 0000000000..5d3aeb112a --- /dev/null +++ b/validation/rancherupgrade/rancherupgrade.go @@ -0,0 +1,29 @@ +// Package rancherupgrade validates the Rancher server upgrade path (2.14.x -> latest) +// with interoperability products installed. Each phase (pre-upgrade, post-upgrade, +// chart-upgrade) is a suite test method selected via the -run regex and runs as its +// own go test invocation, so all cross-phase state lives in-cluster under the +// deterministic names defined here. +package rancherupgrade + +const ( + // UpgradeInputConfigKey is the cattle-config key holding upgrade-phase inputs. + UpgradeInputConfigKey = "rancherUpgradeInput" + + // PipelineConfigKey marks a config as authored by the rancher-upgrade + // pipeline; generic PIT legs never set it. + PipelineConfigKey = "rancherUpgradePipeline" +) + +// UpgradeInput holds per-run upgrade parameters. TargetVersion is the Rancher +// version the server is upgraded to; written into the config by the pipeline +// (qa-infra-automation resolver output) or by hand for local runs. +type UpgradeInput struct { + TargetVersion string `json:"targetVersion" yaml:"targetVersion"` +} + +// PipelineInput is the marker section the rancher-upgrade pipeline writes into +// its cattle-config; the suite fails rather than skips when it is present but +// the upgrade target is missing (pipeline bug, not a generic PIT leg). +type PipelineInput struct { + Enabled bool `json:"enabled" yaml:"enabled"` +} diff --git a/validation/rancherupgrade/schemas/pit_schemas.yaml b/validation/rancherupgrade/schemas/pit_schemas.yaml new file mode 100644 index 0000000000..fea4abbc2d --- /dev/null +++ b/validation/rancherupgrade/schemas/pit_schemas.yaml @@ -0,0 +1,135 @@ +- projects: [RANCHERINT] + suite: Rancher Upgrade/Longhorn + cases: + - title: "Longhorn pre-upgrade setup" + description: "Install Longhorn and seed a checksum workload on a Longhorn volume before the Rancher server upgrade" + automation: 2 + steps: + - action: "Create a project on the Rancher local cluster" + data: "" + expectedresult: "The project is created successfully" + position: 1 + - action: "Install the Longhorn chart at the latest version offered by the pre-upgrade Rancher" + data: "Chart: longhorn from the rancher-charts repo" + expectedresult: "Longhorn installs and its deployments and daemonsets become Ready" + position: 2 + - action: "Create the checksum workload on a Longhorn-backed PVC" + data: "1GiB PVC and nginx deployment named rancher-upgrade-longhorn with data file /auto-mnt/checksum-data" + expectedresult: "PVC is Bound, deployment rolls out, and the checksum data is written to the Longhorn volume" + position: 3 + custom_field: + "15": "TestLonghornPreUpgrade" + - title: "Longhorn post-upgrade validation" + description: "Verify Longhorn and its volume data survive the Rancher server upgrade" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Check the Longhorn chart installation state" + data: "Chart: longhorn in longhorn-system" + expectedresult: "The chart is still installed" + position: 2 + - action: "Verify the checksum workload on the Longhorn volume" + data: "PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" + expectedresult: "PVC is Bound and the sha256 of the data file matches the pre-upgrade content" + position: 3 + custom_field: + "15": "TestLonghornPostUpgrade" + - title: "Longhorn chart upgrade after Rancher upgrade" + description: "Upgrade Longhorn to the version offered by the upgraded Rancher catalog and verify workload data" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Resolve the latest Longhorn chart version from the cluster-scoped catalog" + data: "Chart: longhorn from the rancher-charts repo" + expectedresult: "The latest cluster-scoped chart version is resolved and differs from the installed pre-upgrade version" + position: 2 + - action: "Upgrade the Longhorn chart with retry" + data: "longhorn and longhorn-crd to the latest version" + expectedresult: "The longhorn chart reaches Deployed state and workloads become Ready" + position: 3 + - action: "Verify the chart version and the checksum workload" + data: "Chart version equals latest; PVC rancher-upgrade-longhorn and data file /auto-mnt/checksum-data" + expectedresult: "Installed chart version matches latest and the sha256 of the data file still matches the pre-upgrade content" + position: 4 + custom_field: + "15": "TestLonghornChartUpgrade" + +- projects: [RANCHERINT] + suite: Rancher Upgrade/NeuVector + cases: + - title: "NeuVector pre-upgrade setup" + description: "Install NeuVector and prove admission control enforcement before the Rancher server upgrade" + automation: 2 + steps: + - action: "Resolve the System project on the Rancher local cluster" + data: "Project: System" + expectedresult: "The System project is resolved for the chart install" + position: 1 + - action: "Install the NeuVector chart at the latest version offered by the pre-upgrade Rancher" + data: "Chart: neuvector and neuvector-crd from the rancher-charts repo in cattle-neuvector-system" + expectedresult: "NeuVector installs and its deployments and daemonsets become Ready" + position: 2 + - action: "Enable the NeuVector admission control deny rule" + data: "NvAdmissionControlSecurityRule named local in protect mode denying workload creates in namespace rancher-upgrade-neuvector-deny" + expectedresult: "The custom resource is applied and the neuvector-validating-admission-webhook ValidatingWebhookConfiguration exists" + position: 3 + - action: "Prove the admission webhook denies a known-bad create" + data: "Pod create in namespace rancher-upgrade-neuvector-deny" + expectedresult: "The create is rejected with an admission webhook denial" + position: 4 + custom_field: + "15": "TestNeuVectorPreUpgrade" + - title: "NeuVector post-upgrade validation" + description: "Verify NeuVector services, manager UI and admission enforcement survive the Rancher server upgrade" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Check the NeuVector chart installation state" + data: "Chart: neuvector in cattle-neuvector-system" + expectedresult: "The chart is still installed" + position: 2 + - action: "Verify the NeuVector services are active" + data: "neuvector-service-webui, neuvector-svc-controller, neuvector-svc-admission-webhook" + expectedresult: "All expected NeuVector services exist and are active" + position: 3 + - action: "Verify the NeuVector manager web UI is reachable" + data: "Rancher service proxy path for neuvector-service-webui port 8443" + expectedresult: "The manager web UI responds through the Rancher service proxy" + position: 4 + - action: "Verify the admission webhook still denies the known-bad create" + data: "Pod create in namespace rancher-upgrade-neuvector-deny" + expectedresult: "The create is rejected with an admission webhook denial" + position: 5 + custom_field: + "15": "TestNeuVectorPostUpgrade" + - title: "NeuVector chart upgrade after Rancher upgrade" + description: "Upgrade NeuVector to the version offered by the upgraded Rancher catalog and re-verify health and enforcement" + automation: 2 + steps: + - action: "Wait for the Rancher server to report the upgraded version" + data: "rancherUpgradeInput.targetVersion" + expectedresult: "The server-version setting equals the target version" + position: 1 + - action: "Resolve the latest NeuVector chart version from the cluster-scoped catalog" + data: "Chart: neuvector from the rancher-charts repo" + expectedresult: "The latest cluster-scoped chart version is resolved and differs from the installed pre-upgrade version" + position: 2 + - action: "Upgrade the NeuVector chart with retry, preserving the installed release values" + data: "neuvector and neuvector-crd to the latest version, reusing the installed App values" + expectedresult: "The neuvector chart reaches Deployed state and workloads become Ready" + position: 3 + - action: "Verify the chart version, service health and admission enforcement" + data: "Chart version equals latest; NeuVector services active; pod create in namespace rancher-upgrade-neuvector-deny denied" + expectedresult: "Installed chart version matches latest, all expected services are active and the admission webhook still denies the create" + position: 4 + custom_field: + "15": "TestNeuVectorChartUpgrade"