diff --git a/internal/state/driver_cleanup_test.go b/internal/state/driver_cleanup_test.go new file mode 100644 index 0000000000..2f711960cd --- /dev/null +++ b/internal/state/driver_cleanup_test.go @@ -0,0 +1,573 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package state + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +// ownedByDriver sets a controller OwnerReference to an NVIDIADriver named owner, +// matching what the controller sets and what the production field index reads. +func ownedByDriver(ds *appsv1.DaemonSet, owner string) { + ds.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: nvidiav1alpha1.SchemeGroupVersion.String(), + Kind: nvidiav1alpha1.NVIDIADriverCRDName, + Name: owner, + UID: types.UID("uid-" + owner), + Controller: ptr.To(true), + }} +} + +func makeDaemonSet(name, owner string, desired, misscheduled int32, nodeSelector map[string]string) *appsv1.DaemonSet { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test-operator", + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{NodeSelector: nodeSelector}, + }, + }, + Status: appsv1.DaemonSetStatus{ + DesiredNumberScheduled: desired, + NumberMisscheduled: misscheduled, + }, + } + if owner != "" { + ownedByDriver(ds, owner) + } + return ds +} + +// nvidiaDriverControllerIndex reproduces the production field-index function +// (controllers/nvidiadriver_controller.go): a DaemonSet is indexed by the name +// of its controlling NVIDIADriver, validating the owner's APIVersion and Kind. +func nvidiaDriverControllerIndex(o client.Object) []string { + owner := metav1.GetControllerOf(o.(*appsv1.DaemonSet)) + if owner == nil || + owner.APIVersion != nvidiav1alpha1.SchemeGroupVersion.String() || + owner.Kind != nvidiav1alpha1.NVIDIADriverCRDName { + return nil + } + return []string{owner.Name} +} + +// daemonSetOwnerIndexClient builds a fake client that indexes DaemonSets by their +// controlling NVIDIADriver, matching the field selector used by cleanupStaleDriverDaemonsets. +func daemonSetOwnerIndexClient(sch *runtime.Scheme, objs ...client.Object) client.Client { + return fake.NewClientBuilder(). + WithScheme(sch). + WithObjects(objs...). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, nvidiaDriverControllerIndex). + Build() +} + +func TestCleanupStaleDriverDaemonsets(t *testing.T) { + sch := driverTestScheme(t) + + matchingNode := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "match-node", + Labels: map[string]string{"pool": "gold"}, + }} + + // dsDesired: in desired list and active (Desired>0) -> kept. + dsDesired := makeDaemonSet("ds-desired", "driver-a", 1, 0, nil) + // dsStale: NOT in desired list -> deleted. + dsStale := makeDaemonSet("ds-stale", "driver-a", 0, 0, nil) + // dsInactive: in desired list, Desired=0, selector matches no nodes -> deleted. + dsInactive := makeDaemonSet("ds-inactive", "driver-a", 0, 0, map[string]string{"pool": "silver"}) + // dsMisscheduled: in desired list, Desired=0 but Misscheduled>0 with no matching + // nodes -> kept (NumberMisscheduled==0 is a deletion prerequisite). + dsMisscheduled := makeDaemonSet("ds-misscheduled", "driver-a", 0, 1, map[string]string{"pool": "silver"}) + // dsInactiveButNodes: in desired list, Desired=0, but selector matches a node -> kept. + dsInactiveButNodes := makeDaemonSet("ds-inactive-nodes", "driver-a", 0, 0, map[string]string{"pool": "gold"}) + // dsDriverB: owned by a different NVIDIADriver -> not indexed for driver-a -> kept. + dsDriverB := makeDaemonSet("ds-driver-b", "driver-b", 0, 0, nil) + // dsUnowned: no controller reference -> not indexed -> kept. + dsUnowned := makeDaemonSet("ds-unowned", "", 0, 0, nil) + // dsWrongKind: controlled by driver-a's name but a non-NVIDIADriver kind -> not indexed -> kept. + dsWrongKind := makeDaemonSet("ds-wrong-kind", "", 0, 0, nil) + dsWrongKind.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "apps/v1", Kind: "Deployment", Name: "driver-a", Controller: ptr.To(true), + }} + + cl := daemonSetOwnerIndexClient(sch, matchingNode, + dsDesired, dsStale, dsInactive, dsMisscheduled, dsInactiveButNodes, dsDriverB, dsUnowned, dsWrongKind) + sd := newTestStateDriver(t, cl, sch) + + cr := &nvidiav1alpha1.NVIDIADriver{ObjectMeta: metav1.ObjectMeta{Name: "driver-a"}} + + desiredObjs := []*unstructured.Unstructured{ + newDaemonSetUnstructured("ds-desired", "test-operator"), + newDaemonSetUnstructured("ds-inactive", "test-operator"), + newDaemonSetUnstructured("ds-misscheduled", "test-operator"), + newDaemonSetUnstructured("ds-inactive-nodes", "test-operator"), + } + + require.NoError(t, sd.cleanupStaleDriverDaemonsets(context.Background(), cr, desiredObjs)) + + assertExists := func(name string, shouldExist bool) { + daemonSet := &appsv1.DaemonSet{} + err := cl.Get(context.Background(), types.NamespacedName{Name: name, Namespace: "test-operator"}, daemonSet) + if shouldExist { + require.NoError(t, err, "expected %s to exist", name) + } else { + require.True(t, apierrors.IsNotFound(err), "expected %s to be deleted (NotFound), got: %v", name, err) + } + } + + assertExists("ds-desired", true) + assertExists("ds-stale", false) + assertExists("ds-inactive", false) + assertExists("ds-misscheduled", true) + assertExists("ds-inactive-nodes", true) + // Isolation: cleanup for driver-a must never touch DaemonSets it does not own. + assertExists("ds-driver-b", true) + assertExists("ds-unowned", true) + assertExists("ds-wrong-kind", true) +} + +func TestCleanupStaleDriverDaemonsetsListError(t *testing.T) { + sch := driverTestScheme(t) + errInjected := errors.New("injected list error") + cl := fake.NewClientBuilder().WithScheme(sch). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, nvidiaDriverControllerIndex). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*appsv1.DaemonSetList); ok { + return errInjected + } + return cl.List(ctx, list, opts...) + }, + }).Build() + sd := newTestStateDriver(t, cl, sch) + + cr := &nvidiav1alpha1.NVIDIADriver{ObjectMeta: metav1.ObjectMeta{Name: "driver-a"}} + err := sd.cleanupStaleDriverDaemonsets(context.Background(), cr, nil) + require.ErrorIs(t, err, errInjected) + require.ErrorContains(t, err, "failed to list all NVIDIA driver DaemonSets") +} + +// TestCleanupStaleDriverDaemonsetsNotFoundOnDeleteIgnored covers both delete +// paths (stale and inactive): a DaemonSet vanishing between List and Delete +// returns NotFound, which cleanup treats as successful. +func TestCleanupStaleDriverDaemonsetsNotFoundOnDeleteIgnored(t *testing.T) { + sch := driverTestScheme(t) + cr := &nvidiav1alpha1.NVIDIADriver{ObjectMeta: metav1.ObjectMeta{Name: "driver-a"}} + + // run confirms cleanup actually attempts the delete (exactly once, on the right + // object) and treats the resulting NotFound as success. + run := func(t *testing.T, ds *appsv1.DaemonSet, desired []*unstructured.Unstructured) { + t.Helper() + deleteCalls := 0 + cl := fake.NewClientBuilder().WithScheme(sch).WithObjects(ds). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, nvidiaDriverControllerIndex). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func(_ context.Context, _ client.WithWatch, obj client.Object, _ ...client.DeleteOption) error { + deleteCalls++ + assert.Equal(t, ds.Name, obj.GetName()) + assert.Equal(t, "test-operator", obj.GetNamespace()) + return apierrors.NewNotFound(schema.GroupResource{Group: "apps", Resource: "daemonsets"}, obj.GetName()) + }, + }).Build() + sd := newTestStateDriver(t, cl, sch) + require.NoError(t, sd.cleanupStaleDriverDaemonsets(context.Background(), cr, desired)) + assert.Equal(t, 1, deleteCalls, "expected exactly one delete attempt") + } + + t.Run("stale daemonset", func(t *testing.T) { + // Empty desired list -> ds-stale is not desired -> stale delete path. + run(t, makeDaemonSet("ds-stale", "driver-a", 0, 0, nil), nil) + }) + + t.Run("inactive daemonset", func(t *testing.T) { + // ds-inactive is desired, Desired=0, selector matches no nodes -> inactive delete path. + ds := makeDaemonSet("ds-inactive", "driver-a", 0, 0, map[string]string{"pool": "silver"}) + desired := []*unstructured.Unstructured{newDaemonSetUnstructured("ds-inactive", "test-operator")} + run(t, ds, desired) + }) +} + +func TestGetDriverAdditionalConfigsCertAndKernelAndTopology(t *testing.T) { + certCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "cert-config", Namespace: "test-ns"}, + Data: map[string]string{"ca.crt": "cert-data"}, + } + kernelCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "kernel-config", Namespace: "test-ns"}, + Data: map[string]string{"module.conf": "options nvidia"}, + } + + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).WithObjects(certCM, kernelCM).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + + cr := &nvidiav1alpha1.NVIDIADriver{ + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + CertConfig: &nvidiav1alpha1.DriverCertConfigSpec{Name: "cert-config"}, + KernelModuleConfig: &nvidiav1alpha1.KernelModuleConfigSpec{Name: "kernel-config"}, + VirtualTopologyConfig: &nvidiav1alpha1.VirtualTopologyConfigSpec{ + Name: "topology-config", + }, + }, + } + + configs, err := sd.getDriverAdditionalConfigs( + context.Background(), + cr, + fakeClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "rhel", osVersion: "9.4"}, + ) + require.NoError(t, err) + + mounts := map[string]corev1.VolumeMount{} + for _, m := range configs.VolumeMounts { + mounts[m.Name] = m + } + volumes := map[string]corev1.Volume{} + for _, v := range configs.Volumes { + volumes[v.Name] = v + } + + certDir, err := getCertConfigPath("rhel") + require.NoError(t, err) + + // Each ConfigMap-backed config must produce a read-only per-file mount at the + // expected path plus a matching ConfigMap volume with a Key->Path item. + assertConfigMapMount := func(name, mountDir, file string) { + t.Helper() + m, ok := mounts[name] + require.True(t, ok, "%s mount missing", name) + assert.Equal(t, filepath.Join(mountDir, file), m.MountPath) + assert.Equal(t, file, m.SubPath) + assert.True(t, m.ReadOnly) + + v, ok := volumes[name] + require.True(t, ok, "%s volume missing", name) + require.NotNil(t, v.ConfigMap) + assert.Equal(t, name, v.ConfigMap.Name) + assert.Contains(t, v.ConfigMap.Items, corev1.KeyToPath{Key: file, Path: file}) + } + + assertConfigMapMount("cert-config", certDir, "ca.crt") + assertConfigMapMount("kernel-config", "/drivers", "module.conf") + + // Topology mounts a single well-known file, read-only, from its ConfigMap. + topo, ok := mounts["topology-config"] + require.True(t, ok, "topology-config mount missing") + assert.Equal(t, consts.VGPUTopologyConfigMountPath, topo.MountPath) + assert.Equal(t, consts.VGPUTopologyConfigFileName, topo.SubPath) + assert.True(t, topo.ReadOnly) + topoVol, ok := volumes["topology-config"] + require.True(t, ok, "topology-config volume missing") + require.NotNil(t, topoVol.ConfigMap) + assert.Equal(t, "topology-config", topoVol.ConfigMap.Name) +} + +func TestGetDriverAdditionalConfigsSLESSubscription(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + + cr := &nvidiav1alpha1.NVIDIADriver{} + + configs, err := sd.getDriverAdditionalConfigs( + context.Background(), + cr, + fakeClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "sles", osVersion: "15.5"}, + ) + require.NoError(t, err) + assert.True(t, hasSubscriptionVolumeMount(configs.VolumeMounts), "expected SLES subscription mounts") +} + +func TestGetDriverAdditionalConfigsUnsupportedCertOS(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + + cr := &nvidiav1alpha1.NVIDIADriver{ + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + CertConfig: &nvidiav1alpha1.DriverCertConfigSpec{Name: "cert-config"}, + }, + } + + _, err := sd.getDriverAdditionalConfigs( + context.Background(), + cr, + fakeClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "unsupported-os", osVersion: "1.0"}, + ) + require.ErrorContains(t, err, "not supported") +} + +func TestHandleDefaultImagesInObjectsReRender(t *testing.T) { + sch := driverTestScheme(t) + + sd := newTestStateDriver(t, nil, sch) + + renderData := getMinimalDriverRenderData() + renderData.Runtime.Namespace = "test-operator" + + desiredObjs, err := sd.renderManifestObjects(context.Background(), renderData) + require.NoError(t, err) + + desiredDs, err := getDaemonsetFromObjects(desiredObjs) + require.NoError(t, err) + + // Capture the manager image baked into the freshly-rendered (desired) DaemonSet. + // The "spec changed" branch must return these desired objects unchanged. + expectedManagerImage := managerImageFromDaemonSet(desiredDs) + require.NotEmpty(t, expectedManagerImage) + require.NotEqual(t, "old-manager-image:1.0", expectedManagerImage) + + // Seed a current DaemonSet with a *different* k8s-driver-manager image and a + // stale hash annotation so the re-render path executes and detects a change. + currentDs := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: desiredDs.Name, + Namespace: desiredDs.Namespace, + Annotations: map[string]string{consts.NvidiaAnnotationHashKey: "stale-hash"}, + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + {Name: "k8s-driver-manager", Image: "old-manager-image:1.0"}, + }, + }, + }, + }, + } + + cl := fake.NewClientBuilder().WithScheme(sch).WithObjects(currentDs).Build() + sd.client = cl + + cr := newDriverCR("driver-a") + cr.Spec.Manager.Image = "" // force env-var / default-image handling + + got, err := sd.handleDefaultImagesInObjects(context.Background(), desiredObjs, cr, *renderData) + require.NoError(t, err) + require.NotEmpty(t, got) + + // The driver spec effectively changed (stale hash != freshly computed hash), so the + // function must keep the desired objects, i.e. the NEW manager image, and must NOT + // downgrade to the current DaemonSet's "old-manager-image:1.0". + gotDs, err := getDaemonsetFromObjects(got) + require.NoError(t, err) + assert.Equal(t, expectedManagerImage, managerImageFromDaemonSet(gotDs)) + assert.NotEqual(t, "old-manager-image:1.0", managerImageFromDaemonSet(gotDs)) +} + +// managerImageFromDaemonSet returns the image of the k8s-driver-manager init container. +func managerImageFromDaemonSet(daemonSet *appsv1.DaemonSet) string { + for _, c := range daemonSet.Spec.Template.Spec.InitContainers { + if c.Name == "k8s-driver-manager" { + return c.Image + } + } + return "" +} + +// clientScheme returns a scheme with core types registered for volume-config tests. +func clientScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(s)) + return s +} + +func TestGetDriverAdditionalConfigsRepoConfigUnsupportedOS(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + cr := &nvidiav1alpha1.NVIDIADriver{ + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + RepoConfig: &nvidiav1alpha1.DriverRepoConfigSpec{Name: "repo-config"}, + }, + } + _, err := sd.getDriverAdditionalConfigs(context.Background(), cr, + fakeClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "unsupported-os", osVersion: "1.0"}) + require.ErrorContains(t, err, "custom repo config") +} + +func TestGetDriverAdditionalConfigsRepoConfigMissingConfigMap(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + cr := &nvidiav1alpha1.NVIDIADriver{ + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + RepoConfig: &nvidiav1alpha1.DriverRepoConfigSpec{Name: "missing-repo"}, + }, + } + _, err := sd.getDriverAdditionalConfigs(context.Background(), cr, + fakeClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "ubuntu", osVersion: "22.04"}) + require.ErrorContains(t, err, "custom repo config") +} + +func TestGetDriverAdditionalConfigsCertConfigMissingConfigMap(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + cr := &nvidiav1alpha1.NVIDIADriver{ + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + CertConfig: &nvidiav1alpha1.DriverCertConfigSpec{Name: "missing-cert"}, + }, + } + _, err := sd.getDriverAdditionalConfigs(context.Background(), cr, + fakeClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "ubuntu", osVersion: "22.04"}) + require.ErrorContains(t, err, "custom certs") +} + +func TestGetDriverAdditionalConfigsKernelModuleMissingConfigMap(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + cr := &nvidiav1alpha1.NVIDIADriver{ + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + KernelModuleConfig: &nvidiav1alpha1.KernelModuleConfigSpec{Name: "missing-kmod"}, + }, + } + _, err := sd.getDriverAdditionalConfigs(context.Background(), cr, + fakeClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "ubuntu", osVersion: "22.04"}) + require.ErrorContains(t, err, "kernel module configuration") +} + +func TestGetDriverAdditionalConfigsRuntimeError(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + cr := &nvidiav1alpha1.NVIDIADriver{} + _, err := sd.getDriverAdditionalConfigs(context.Background(), cr, + fakeClusterInfo{runtimeErr: fmt.Errorf("runtime boom")}, + nodePool{osRelease: "ubuntu", osVersion: "22.04"}) + require.ErrorContains(t, err, "retrieve container runtime") +} + +func TestGetDriverAdditionalConfigsOpenshiftVersionError(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + cr := &nvidiav1alpha1.NVIDIADriver{} + _, err := sd.getDriverAdditionalConfigs(context.Background(), cr, + fakeClusterInfo{runtime: consts.Containerd, openshiftVersionErr: fmt.Errorf("ocp boom")}, + nodePool{osRelease: "ubuntu", osVersion: "22.04"}) + require.ErrorContains(t, err, "introspecting cluster") +} + +func volumeByName(t *testing.T, vols []corev1.Volume, name string) corev1.Volume { + t.Helper() + for _, v := range vols { + if v.Name == name { + return v + } + } + t.Fatalf("volume %q not found", name) + return corev1.Volume{} +} + +// licensingMountsByPath indexes the "licensing-config" volume mounts by mount path. +func licensingMountsByPath(mounts []corev1.VolumeMount) map[string]corev1.VolumeMount { + out := map[string]corev1.VolumeMount{} + for _, m := range mounts { + if m.Name == "licensing-config" { + out[m.MountPath] = m + } + } + return out +} + +func TestGetDriverAdditionalConfigsLicensingConfigMap(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + cr := &nvidiav1alpha1.NVIDIADriver{ + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + // Name set, no SecretName, NLSEnabled defaults to true. + LicensingConfig: &nvidiav1alpha1.DriverLicensingConfigSpec{Name: "lic-config"}, + }, + } + configs, err := sd.getDriverAdditionalConfigs(context.Background(), cr, + fakeClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "ubuntu", osVersion: "22.04"}) + require.NoError(t, err) + + vol := volumeByName(t, configs.Volumes, "licensing-config") + require.NotNil(t, vol.ConfigMap, "NLS-enabled licensing should use a ConfigMap volume") + assert.Equal(t, "lic-config", vol.ConfigMap.Name) + assert.Nil(t, vol.Secret) + + mounts := licensingMountsByPath(configs.VolumeMounts) + // gridd.conf is always mounted read-only at the licensing path... + gridd, ok := mounts[consts.VGPULicensingConfigMountPath] + require.True(t, ok, "gridd.conf licensing mount missing") + assert.Equal(t, consts.VGPULicensingFileName, gridd.SubPath) + assert.True(t, gridd.ReadOnly) + // ...and with NLS enabled the client token is mounted too. + token, ok := mounts[consts.NLSClientTokenMountPath] + require.True(t, ok, "NLS client-token mount missing when NLS is enabled") + assert.Equal(t, consts.NLSClientTokenFileName, token.SubPath) + assert.True(t, token.ReadOnly) +} + +func TestGetDriverAdditionalConfigsLicensingSecretNoNLS(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(clientScheme(t)).Build() + sd := &stateDriver{stateSkel: stateSkel{client: cl, namespace: "test-ns"}} + cr := &nvidiav1alpha1.NVIDIADriver{ + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + LicensingConfig: &nvidiav1alpha1.DriverLicensingConfigSpec{ + SecretName: "lic-secret", + NLSEnabled: ptr.To(false), + }, + }, + } + configs, err := sd.getDriverAdditionalConfigs(context.Background(), cr, + fakeClusterInfo{runtime: consts.Containerd}, + nodePool{osRelease: "ubuntu", osVersion: "22.04"}) + require.NoError(t, err) + + vol := volumeByName(t, configs.Volumes, "licensing-config") + require.NotNil(t, vol.Secret, "licensing should use a Secret volume") + assert.Equal(t, "lic-secret", vol.Secret.SecretName) + assert.Nil(t, vol.ConfigMap) + + mounts := licensingMountsByPath(configs.VolumeMounts) + gridd, ok := mounts[consts.VGPULicensingConfigMountPath] + require.True(t, ok, "gridd.conf licensing mount missing") + assert.Equal(t, consts.VGPULicensingFileName, gridd.SubPath) + assert.True(t, gridd.ReadOnly) + // NLS disabled -> no client-token mount. + _, hasToken := mounts[consts.NLSClientTokenMountPath] + assert.False(t, hasToken, "NLS token must not be mounted when NLS is disabled") +} diff --git a/internal/state/driver_manifest_test.go b/internal/state/driver_manifest_test.go new file mode 100644 index 0000000000..6d5ee1a358 --- /dev/null +++ b/internal/state/driver_manifest_test.go @@ -0,0 +1,1028 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package state + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + "github.com/go-logr/logr/funcr" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + + nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" + driverconfig "github.com/NVIDIA/gpu-operator/internal/config" + "github.com/NVIDIA/gpu-operator/internal/consts" + "github.com/NVIDIA/gpu-operator/internal/utils" +) + +// coreAppsScheme returns a scheme with only core and apps types registered +// (no NVIDIADriver), used to trigger SetControllerReference errors. +func coreAppsScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(s)) + require.NoError(t, appsv1.AddToScheme(s)) + return s +} + +func fullCatalog() InfoCatalog { + catalog := NewInfoCatalog() + catalog.Add(InfoTypeHostRoot, "/host") + catalog.Add(InfoTypeClusterInfo, fakeClusterInfo{}) + return catalog +} + +// daemonSetsByName converts every DaemonSet object in objs to a typed value, +// keyed by name. +func daemonSetsByName(t *testing.T, objs []*unstructured.Unstructured) map[string]*appsv1.DaemonSet { + t.Helper() + out := map[string]*appsv1.DaemonSet{} + for _, o := range objs { + if o.GetKind() != "DaemonSet" { + continue + } + ds := &appsv1.DaemonSet{} + require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(o.Object, ds)) + out[ds.Name] = ds + } + return out +} + +// requireDS returns the single DaemonSet whose name has the given prefix, failing +// if zero or more than one match (names carry a nondeterministic hash suffix). +func requireDS(t *testing.T, byName map[string]*appsv1.DaemonSet, prefix string) *appsv1.DaemonSet { + t.Helper() + var found *appsv1.DaemonSet + for name, ds := range byName { + if strings.HasPrefix(name, prefix) { + require.Nil(t, found, "multiple DaemonSets match prefix %q", prefix) + found = ds + } + } + require.NotNil(t, found, "no DaemonSet matched prefix %q", prefix) + return found +} + +// newGPUNodeOS builds a GPU node advertising a specific OS release/version. +func newGPUNodeOS(name, owner, osID, osVersion string) *corev1.Node { + return &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + consts.GPUPresentLabel: "true", + consts.NVIDIADriverOwnerLabel: owner, + nfdOSReleaseIDLabelKey: osID, + nfdOSVersionIDLabelKey: osVersion, + }, + }} +} + +// --- NewStateDriver error path ------------------------------------------------- + +func TestNewStateDriverBadManifestDir(t *testing.T) { + _, err := NewStateDriver(nil, "", nil, "/nonexistent/manifest/dir") + require.ErrorContains(t, err, "failed to get files from manifest directory") +} + +// --- getDriverName truncation -------------------------------------------------- + +func TestGetDriverNameTruncation(t *testing.T) { + cr := &nvidiav1alpha1.NVIDIADriver{ + ObjectMeta: metav1.ObjectMeta{Name: strings.Repeat("a", 300)}, + Spec: nvidiav1alpha1.NVIDIADriverSpec{DriverType: nvidiav1alpha1.GPU}, + } + name := getDriverName(cr, "ubuntu22.04") + + // "nvidia-gpu-driver-" (18 chars) + the CR name, truncated to 253. + expected := "nvidia-gpu-driver-" + strings.Repeat("a", 253-len("nvidia-gpu-driver-")) + assert.Equal(t, expected, name) + assert.Len(t, name, 253) + // The truncated name must remain a valid Kubernetes object name. + assert.Empty(t, validation.IsDNS1123Subdomain(name)) + // Deterministic for identical input. + assert.Equal(t, name, getDriverName(cr, "ubuntu22.04")) +} + +// --- startup probe defaults ---------------------------------------------------- + +func TestGetDefaultStartupProbe(t *testing.T) { + testCases := []struct { + name string + precompiled bool + wantInitial int32 + }{ + {"standard driver uses the longer 60s delay", false, 60}, + {"precompiled driver uses the shorter 5s delay", true, 5}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + spec := &nvidiav1alpha1.NVIDIADriverSpec{} + if tc.precompiled { + spec.UsePrecompiled = ptr.To(true) + } + probe := getDefaultStartupProbe(spec) + require.NotNil(t, probe) + assert.Equal(t, tc.wantInitial, probe.InitialDelaySeconds) + // The remaining defaults are shared regardless of driver type. + assert.Equal(t, int32(60), probe.TimeoutSeconds) + assert.Equal(t, int32(10), probe.PeriodSeconds) + assert.Equal(t, int32(1), probe.SuccessThreshold) + assert.Equal(t, int32(120), probe.FailureThreshold) + }) + } +} + +func TestGetDriverSpecPreservesUserStartupProbe(t *testing.T) { + cr := newDriverCR("driver-a") + custom := &nvidiav1alpha1.ContainerProbeSpec{ + InitialDelaySeconds: 7, TimeoutSeconds: 3, PeriodSeconds: 2, SuccessThreshold: 1, FailureThreshold: 9, + } + cr.Spec.StartupProbe = custom + + spec, err := getDriverSpec(cr, nodePool{osTag: "ubuntu22.04"}) + require.NoError(t, err) + // A user-provided probe must be preserved, not replaced by the defaults. + assert.Equal(t, custom, spec.Spec.StartupProbe) +} + +// --- getDriverSpec manager image error ----------------------------------------- + +func TestGetDriverSpecManagerImageError(t *testing.T) { + // Ensure the fallback env var is not set so an empty Manager image errors. + t.Setenv("DRIVER_MANAGER_IMAGE", "") + cr := &nvidiav1alpha1.NVIDIADriver{ + ObjectMeta: metav1.ObjectMeta{Name: "driver-a"}, + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + DriverType: nvidiav1alpha1.GPU, + Repository: "nvcr.io/nvidia", + Image: "driver", + Version: "535.104.05", + // Manager repository/image/version all empty -> image.ImagePath errors. + Manager: nvidiav1alpha1.DriverManagerSpec{}, + }, + } + _, err := getDriverSpec(cr, nodePool{osTag: "ubuntu22.04"}) + require.ErrorContains(t, err, "failed to construct image path for driver manager") +} + +// --- getObjectOfKind / getDaemonsetFromObjects errors -------------------------- + +func TestGetObjectOfKindNotFound(t *testing.T) { + _, err := getObjectOfKind([]*unstructured.Unstructured{}, "DaemonSet") + require.ErrorContains(t, err, "did not find object of kind") +} + +func TestGetDaemonsetFromObjectsErrors(t *testing.T) { + // No DaemonSet present. + _, err := getDaemonsetFromObjects([]*unstructured.Unstructured{newConfigMapUnstructured("cm", "ns")}) + require.ErrorContains(t, err, "did not find object of kind") + + // A DaemonSet-kinded object whose nested fields have the wrong type -> conversion error. + bad := newDaemonSetUnstructured("ds-bad", "ns") + bad.Object["spec"] = "not-a-spec-object" + _, err = getDaemonsetFromObjects([]*unstructured.Unstructured{bad}) + require.ErrorContains(t, err, "error converting unstructured object to DaemonSet") +} + +// --- renderManifestObjects error path ------------------------------------------ + +func TestRenderManifestObjectsError(t *testing.T) { + state, err := NewStateDriver(nil, "", nil, manifestDir) + require.NoError(t, err) + sd := state.(*stateDriver) + + // Empty render data: templates dereference .Driver.Spec fields, which are nil, + // causing template execution to fail. + _, err = sd.renderManifestObjects(context.Background(), &driverRenderData{}) + require.Error(t, err) +} + +// --- getManifestObjects error/branch coverage ---------------------------------- + +func TestGetManifestObjectsRuntimeSpecError(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch) + sd := newTestStateDriver(t, cl, sch) + + catalog := NewInfoCatalog() + catalog.Add(InfoTypeHostRoot, "/host") + catalog.Add(InfoTypeClusterInfo, fakeClusterInfo{openshiftVersionErr: fmt.Errorf("boom")}) + + _, err := sd.getManifestObjects(context.Background(), newDriverCR("driver-a"), catalog) + require.ErrorContains(t, err, "failed to construct cluster runtime spec") +} + +func TestGetManifestObjectsNodeListError(t *testing.T) { + sch := driverTestScheme(t) + errInjected := errors.New("injected node list error") + cl := fake.NewClientBuilder().WithScheme(sch). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, func(_ client.Object) []string { return nil }). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*corev1.NodeList); ok { + return errInjected + } + return cl.List(ctx, list, opts...) + }, + }).Build() + sd := newTestStateDriver(t, cl, sch) + + _, err := sd.getManifestObjects(context.Background(), newDriverCR("driver-a"), fullCatalog()) + require.ErrorIs(t, err, errInjected) + require.ErrorContains(t, err, "failed to get node pools") +} + +func TestGetManifestObjectsHostRootWrongType(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch, newGPUNode("gpu-node", "driver-a")) + sd := newTestStateDriver(t, cl, sch) + + catalog := NewInfoCatalog() + catalog.Add(InfoTypeHostRoot, 123) // present but not a string + catalog.Add(InfoTypeClusterInfo, fakeClusterInfo{}) + + _, err := sd.getManifestObjects(context.Background(), newDriverCR("driver-a"), catalog) + require.ErrorContains(t, err, "host root in info catalog has unexpected type") +} + +func TestGetManifestObjectsDriverSpecError(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch, newGPUNode("gpu-node", "driver-a")) + sd := newTestStateDriver(t, cl, sch) + + cr := newDriverCR("driver-a") + cr.Spec.Image = "INVALID IMAGE" // breaks getDriverImagePath inside getDriverSpec + + _, err := sd.getManifestObjects(context.Background(), cr, fullCatalog()) + require.ErrorContains(t, err, "failed to construct driver spec") +} + +func TestGetManifestObjectsGDSError(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch, newGPUNode("gpu-node", "driver-a")) + sd := newTestStateDriver(t, cl, sch) + + cr := newDriverCR("driver-a") + cr.Spec.GPUDirectStorage = &nvidiav1alpha1.GPUDirectStorageSpec{ + Enabled: ptr.To(true), + Image: "INVALID IMAGE", + } + + _, err := sd.getManifestObjects(context.Background(), cr, fullCatalog()) + require.ErrorContains(t, err, "failed to construct GDS spec") +} + +func TestGetManifestObjectsGDRCopyError(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch, newGPUNode("gpu-node", "driver-a")) + sd := newTestStateDriver(t, cl, sch) + + cr := newDriverCR("driver-a") + cr.Spec.GDRCopy = &nvidiav1alpha1.GDRCopySpec{ + Enabled: ptr.To(true), + Image: "INVALID IMAGE", + } + + _, err := sd.getManifestObjects(context.Background(), cr, fullCatalog()) + require.ErrorContains(t, err, "failed to construct GDRCopy spec") +} + +func TestGetManifestObjectsPrecompiled(t *testing.T) { + sch := driverTestScheme(t) + // A kernel that actually requires sanitization: the arch suffix is stripped + // (and trailing dot trimmed) for the resource-metadata label, while the image + // tag and node selector keep the raw kernel. + const rawKernel = "5.14.0-427.el9.x86_64" + const sanitizedKernel = "5.14.0-427.el9" + node := newGPUNode("gpu-node", "driver-a") + node.Labels[nfdKernelLabelKey] = rawKernel + cl := driverIndexBuilder(sch, node) + sd := newTestStateDriver(t, cl, sch) + + cr := newDriverCR("driver-a") + cr.Spec.UsePrecompiled = ptr.To(true) + + objs, err := sd.getManifestObjects(context.Background(), cr, fullCatalog()) + require.NoError(t, err) + + ds, err := getDaemonsetFromObjects(objs) + require.NoError(t, err) + + // The node selector and image tag pin the exact (raw) kernel... + assert.Equal(t, rawKernel, ds.Spec.Template.Spec.NodeSelector[nfdKernelLabelKey]) + drv := containerByName(t, ds, "nvidia-driver-ctr") + assert.Equal(t, "nvcr.io/nvidia/driver:535.104.05-"+rawKernel+"-ubuntu22.04", drv.Image) + // ...while the precompiled labels mark the branch and carry the sanitized kernel. + assert.Equal(t, "true", ds.Labels["nvidia.com/precompiled"]) + assert.Equal(t, sanitizedKernel, ds.Labels["nvidia.com/precompiled.kernel-version"]) +} + +func TestGetManifestObjectsOpenshiftDTK(t *testing.T) { + sch := driverTestScheme(t) + const rhcosVersion = "413.92.202304252344-0" + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "rhcos-node", + Labels: map[string]string{ + consts.GPUPresentLabel: "true", + consts.NVIDIADriverOwnerLabel: "driver-a", + nfdOSReleaseIDLabelKey: "rhcos", + nfdOSVersionIDLabelKey: "4.13", + nfdOSTreeVersionLabelKey: rhcosVersion, + }, + }} + const dtkImage = "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:7fecaebc1d51b28bc3548171907e4d91823a031d7a6a694ab686999be2b4d867" + cl := driverIndexBuilder(sch, node) + sd := newTestStateDriver(t, cl, sch) + + catalog := NewInfoCatalog() + catalog.Add(InfoTypeHostRoot, "/host") + catalog.Add(InfoTypeClusterInfo, fakeClusterInfo{ + openshiftVersion: "4.13", + dtkImages: map[string]string{rhcosVersion: dtkImage}, + }) + + cr := newDriverCR("driver-a") + objs, err := sd.getManifestObjects(context.Background(), cr, catalog) + require.NoError(t, err) + + ds, err := getDaemonsetFromObjects(objs) + require.NoError(t, err) + + // DTK build path: the RHCOS OSTree selector pins the pool... + assert.Equal(t, rhcosVersion, ds.Spec.Template.Spec.NodeSelector[nfdOSTreeVersionLabelKey]) + // ...the driver-toolkit container carries the resolved DTK image... + dtk := containerByName(t, ds, "openshift-driver-toolkit-ctr") + assert.Equal(t, dtkImage, dtk.Image) + // ...and the DTK labels mark the build path (not the ordinary prebuilt path). + assert.Equal(t, "true", ds.Labels[consts.OcpDriverToolkitIdentificationLabel]) + assert.Equal(t, rhcosVersion, ds.Labels[consts.OcpDriverToolkitVersionLabel]) + + // With a matching DTK image, the missing-image fallback markers must be absent. + assert.NotContains(t, ds.Labels, dtkImageMissingLabel) + _, driverMissing := envValue(containerByName(t, ds, "nvidia-driver-ctr"), "RHCOS_IMAGE_MISSING") + assert.False(t, driverMissing, "driver container must not carry RHCOS_IMAGE_MISSING when a DTK image is found") + _, dtkMissing := envValue(dtk, "RHCOS_IMAGE_MISSING") + assert.False(t, dtkMissing, "DTK container must not carry RHCOS_IMAGE_MISSING when a DTK image is found") +} + +func envValue(c corev1.Container, name string) (string, bool) { + for _, e := range c.Env { + if e.Name == name { + return e.Value, true + } + } + return "", false +} + +const dtkImageMissingLabel = "openshift.driver-toolkit.rhcos-image-missing" + +func TestGetManifestObjectsOpenshiftDTKMissingImageForNodePool(t *testing.T) { + // DTK is enabled whenever the image map is nonempty, and the per-pool image is + // looked up by exact RHCOS version. When that lookup misses, the manifest + // template falls back to the driver image for the driver-toolkit container + // (rather than rendering an empty image), so no invalid DaemonSet is produced. + sch := driverTestScheme(t) + const rhcosVersion = "413.92.202304252344-0" + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "rhcos-node", + Labels: map[string]string{ + consts.GPUPresentLabel: "true", + consts.NVIDIADriverOwnerLabel: "driver-a", + nfdOSReleaseIDLabelKey: "rhcos", + nfdOSVersionIDLabelKey: "4.13", + nfdOSTreeVersionLabelKey: rhcosVersion, + }, + }} + sd := newTestStateDriver(t, driverIndexBuilder(sch, node), sch) + + catalog := NewInfoCatalog() + catalog.Add(InfoTypeHostRoot, "/host") + catalog.Add(InfoTypeClusterInfo, fakeClusterInfo{ + openshiftVersion: "4.13", + // Nonempty (so DTK is enabled) but with no entry for this node pool's RHCOS version. + dtkImages: map[string]string{"999.99.99-0": "quay.io/dtk@sha256:other"}, + }) + + objs, err := sd.getManifestObjects(context.Background(), newDriverCR("driver-a"), catalog) + require.NoError(t, err) + + ds, err := getDaemonsetFromObjects(objs) + require.NoError(t, err) + dtk := containerByName(t, ds, "openshift-driver-toolkit-ctr") + driver := containerByName(t, ds, "nvidia-driver-ctr") + assert.NotEmpty(t, dtk.Image, "missing DTK image must not render an empty container image") + assert.Equal(t, driver.Image, dtk.Image, "driver-toolkit falls back to the driver image when no DTK image matches the pool") + + // The fallback also flips the marker label and the RHCOS env in both containers, + // which drive the DTK sidecar to self-build the driver. + assert.Equal(t, "true", ds.Labels[dtkImageMissingLabel]) + assert.Equal(t, "true", ds.Spec.Template.Labels[dtkImageMissingLabel]) + for _, c := range []corev1.Container{driver, dtk} { + missing, ok := envValue(c, "RHCOS_IMAGE_MISSING") + assert.True(t, ok, "%s missing RHCOS_IMAGE_MISSING env", c.Name) + assert.Equal(t, "true", missing) + version, ok := envValue(c, "RHCOS_VERSION") + assert.True(t, ok, "%s missing RHCOS_VERSION env", c.Name) + assert.Equal(t, rhcosVersion, version) + } +} + +func TestGetManifestObjectsMultipleNodePools(t *testing.T) { + sch := driverTestScheme(t) + + t.Run("distinct OS versions render one DaemonSet each", func(t *testing.T) { + cl := driverIndexBuilder(sch, + newGPUNode("u22", "driver-a"), // ubuntu 22.04 + newGPUNodeOS("u20", "driver-a", "ubuntu", "20.04"), // ubuntu 20.04 + ) + sd := newTestStateDriver(t, cl, sch) + + objs, err := sd.getManifestObjects(context.Background(), newDriverCR("driver-a"), fullCatalog()) + require.NoError(t, err) + + // Compare as a set: the render loop appends pools in nondeterministic order. + byName := daemonSetsByName(t, objs) + require.Len(t, byName, 2) + u22 := requireDS(t, byName, "nvidia-gpu-driver-ubuntu22.04-") + u20 := requireDS(t, byName, "nvidia-gpu-driver-ubuntu20.04-") + + assert.Equal(t, "22.04", u22.Spec.Template.Spec.NodeSelector[nfdOSVersionIDLabelKey]) + assert.Equal(t, "20.04", u20.Spec.Template.Spec.NodeSelector[nfdOSVersionIDLabelKey]) + assert.Equal(t, "nvcr.io/nvidia/driver:535.104.05-ubuntu22.04", containerByName(t, u22, "nvidia-driver-ctr").Image) + assert.Equal(t, "nvcr.io/nvidia/driver:535.104.05-ubuntu20.04", containerByName(t, u20, "nvidia-driver-ctr").Image) + }) + + t.Run("two nodes in the same pool render a single DaemonSet", func(t *testing.T) { + cl := driverIndexBuilder(sch, + newGPUNode("u22-a", "driver-a"), + newGPUNode("u22-b", "driver-a"), + ) + sd := newTestStateDriver(t, cl, sch) + + objs, err := sd.getManifestObjects(context.Background(), newDriverCR("driver-a"), fullCatalog()) + require.NoError(t, err) + assert.Len(t, daemonSetsByName(t, objs), 1) + }) +} + +func TestGetManifestObjectsAdditionalConfigsErrorIsLogged(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch, newGPUNode("gpu-node", "driver-a")) + sd := newTestStateDriver(t, cl, sch) + + cr := newDriverCR("driver-a") + // Reference a ConfigMap that does not exist -> getDriverAdditionalConfigs errors. + cr.Spec.RepoConfig = &nvidiav1alpha1.DriverRepoConfigSpec{Name: "missing-repo-config"} + + // Capture the logger to assert the failure is logged and non-fatal, not swallowed. + var logs strings.Builder + logger := funcr.New(func(_, args string) { logs.WriteString(args + "\n") }, funcr.Options{}) + ctx := log.IntoContext(context.Background(), logger) + + objs, err := sd.getManifestObjects(ctx, cr, fullCatalog()) + require.NoError(t, err) + require.NotEmpty(t, objs) + // The log must name the specific ConfigMap that failed to load, not just a generic message. + assert.Contains(t, logs.String(), "error rendering addition driver volume") + assert.Contains(t, logs.String(), "missing-repo-config") + + // Since the requested config could not be resolved, its volume must not be + // mounted; the driver is deployed without the configuration the user asked for. + ds, err := getDaemonsetFromObjects(objs) + require.NoError(t, err) + for _, v := range ds.Spec.Template.Spec.Volumes { + assert.NotEqual(t, "missing-repo-config", v.Name, "unresolved repo config must not be mounted") + } +} + +func TestGetManifestObjectsHandleDefaultImagesError(t *testing.T) { + sch := driverTestScheme(t) + cl := fake.NewClientBuilder().WithScheme(sch). + WithObjects(newGPUNode("gpu-node", "driver-a")). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, func(_ client.Object) []string { return nil }). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*appsv1.DaemonSet); ok { + return fmt.Errorf("injected daemonset get error") + } + return cl.Get(ctx, key, obj, opts...) + }, + }).Build() + sd := newTestStateDriver(t, cl, sch) + + cr := newDriverCR("driver-a") + // The default (env-var) driver-manager image lets rendering succeed, so + // handleDefaultImages then Gets the current DaemonSet to check for an update. + cr.Spec.Manager.Repository = "" + cr.Spec.Manager.Image = "" + cr.Spec.Manager.Version = "" + t.Setenv("DRIVER_MANAGER_IMAGE", "nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.6.2") + + _, err := sd.getManifestObjects(context.Background(), cr, fullCatalog()) + require.ErrorContains(t, err, "failed to get current driver DaemonSet") +} + +// --- Sync error paths ---------------------------------------------------------- + +func TestSyncGetManifestObjectsError(t *testing.T) { + sd := newTestStateDriver(t, nil, driverTestScheme(t)) + + // Empty catalog -> getManifestObjects fails. + syncState, err := sd.Sync(context.Background(), newDriverCR("driver-a"), NewInfoCatalog()) + require.ErrorContains(t, err, "failed to create k8s objects from manifests") + assert.Equal(t, SyncState(SyncStateNotReady), syncState) +} + +func TestSyncCleanupError(t *testing.T) { + sch := driverTestScheme(t) + cl := fake.NewClientBuilder().WithScheme(sch). + WithObjects(newGPUNode("gpu-node", "driver-a")). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, func(_ client.Object) []string { return nil }). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*appsv1.DaemonSetList); ok { + return fmt.Errorf("injected daemonset list error") + } + return cl.List(ctx, list, opts...) + }, + }).Build() + sd := newTestStateDriver(t, cl, sch) + + syncState, err := sd.Sync(context.Background(), newDriverCR("driver-a"), fullCatalog()) + require.ErrorContains(t, err, "failed to cleanup stale driver DaemonSets") + assert.Equal(t, SyncState(SyncStateNotReady), syncState) +} + +func TestSyncCreateOrUpdateError(t *testing.T) { + // Scheme without NVIDIADriver registered -> SetControllerReference fails inside + // createOrUpdateObjs. + sch := coreAppsScheme(t) + cl := fake.NewClientBuilder().WithScheme(sch). + WithObjects(newGPUNode("gpu-node", "driver-a")). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, func(_ client.Object) []string { return nil }). + Build() + sd := newTestStateDriver(t, cl, sch) + + syncState, err := sd.Sync(context.Background(), newDriverCR("driver-a"), fullCatalog()) + require.ErrorContains(t, err, "failed to create/update objects") + assert.Equal(t, SyncState(SyncStateNotReady), syncState) +} + +func TestSyncGetSyncStateError(t *testing.T) { + sch := driverTestScheme(t) + errInjected := errors.New("injected get error") + cl := fake.NewClientBuilder().WithScheme(sch). + WithObjects(newGPUNode("gpu-node", "driver-a")). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, func(_ client.Object) []string { return nil }). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + // Fail only the readiness Gets (unstructured) performed by getSyncState. + if _, ok := obj.(*unstructured.Unstructured); ok { + return errInjected + } + return cl.Get(ctx, key, obj, opts...) + }, + }).Build() + sd := newTestStateDriver(t, cl, sch) + + syncState, err := sd.Sync(context.Background(), newDriverCR("driver-a"), fullCatalog()) + require.ErrorContains(t, err, "failed to get sync state") + assert.Equal(t, SyncState(SyncStateNotReady), syncState) +} + +// --- cleanupStaleDriverDaemonsets delete/list error paths ---------------------- + +func TestCleanupStaleDeleteErrors(t *testing.T) { + sch := driverTestScheme(t) + + t.Run("stale daemonset delete error", func(t *testing.T) { + dsStale := makeDaemonSet("ds-stale", "driver-a", 0, 0, nil) + cl := fake.NewClientBuilder().WithScheme(sch).WithObjects(dsStale). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, nvidiaDriverControllerIndex). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.DeleteOption) error { + return fmt.Errorf("injected delete error") + }, + }).Build() + sd := newTestStateDriver(t, cl, sch) + cr := &nvidiav1alpha1.NVIDIADriver{ObjectMeta: metav1.ObjectMeta{Name: "driver-a"}} + // desiredObjs empty -> dsStale is not desired -> deleted -> delete error. + err := sd.cleanupStaleDriverDaemonsets(context.Background(), cr, nil) + require.ErrorContains(t, err, "error deleting DaemonSet") + }) + + t.Run("node list error", func(t *testing.T) { + dsInactive := makeDaemonSet("ds-inactive", "driver-a", 0, 0, map[string]string{"pool": "gold"}) + cl := fake.NewClientBuilder().WithScheme(sch).WithObjects(dsInactive). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, nvidiaDriverControllerIndex). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*corev1.NodeList); ok { + return fmt.Errorf("injected node list error") + } + return cl.List(ctx, list, opts...) + }, + }).Build() + sd := newTestStateDriver(t, cl, sch) + cr := &nvidiav1alpha1.NVIDIADriver{ObjectMeta: metav1.ObjectMeta{Name: "driver-a"}} + desired := []*unstructured.Unstructured{newDaemonSetUnstructured("ds-inactive", "test-operator")} + err := sd.cleanupStaleDriverDaemonsets(context.Background(), cr, desired) + require.ErrorContains(t, err, "failed to list nodes") + }) + + t.Run("inactive daemonset delete error", func(t *testing.T) { + dsInactive := makeDaemonSet("ds-inactive", "driver-a", 0, 0, map[string]string{"pool": "silver"}) + cl := fake.NewClientBuilder().WithScheme(sch).WithObjects(dsInactive). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, nvidiaDriverControllerIndex). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.DeleteOption) error { + return fmt.Errorf("injected delete error") + }, + }).Build() + sd := newTestStateDriver(t, cl, sch) + cr := &nvidiav1alpha1.NVIDIADriver{ObjectMeta: metav1.ObjectMeta{Name: "driver-a"}} + desired := []*unstructured.Unstructured{newDaemonSetUnstructured("ds-inactive", "test-operator")} + err := sd.cleanupStaleDriverDaemonsets(context.Background(), cr, desired) + require.ErrorContains(t, err, "error deleting DaemonSet") + }) +} + +// --- handleDefaultImagesInObjects additional branches -------------------------- + +func TestHandleDefaultImagesNoDaemonSet(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch) + sd := newTestStateDriver(t, cl, sch) + + cr := newDriverCR("driver-a") + cr.Spec.Manager.Image = "" + renderData := getMinimalDriverRenderData() + + // objs without any DaemonSet -> getDaemonsetFromObjects fails. + objs := []*unstructured.Unstructured{newConfigMapUnstructured("cm", "test-operator")} + _, err := sd.handleDefaultImagesInObjects(context.Background(), objs, cr, *renderData) + require.ErrorContains(t, err, "error getting DaemonSet from unstructured objects") +} + +func TestHandleDefaultImagesCurrentImageMatches(t *testing.T) { + sch := driverTestScheme(t) + + sd := newTestStateDriver(t, nil, sch) + renderData := getMinimalDriverRenderData() + renderData.Runtime.Namespace = "test-operator" + desiredObjs, err := sd.renderManifestObjects(context.Background(), renderData) + require.NoError(t, err) + desiredDs, err := getDaemonsetFromObjects(desiredObjs) + require.NoError(t, err) + + // Current DaemonSet already runs the same k8s-driver-manager image. + currentDs := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: desiredDs.Name, Namespace: desiredDs.Namespace}, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + {Name: "k8s-driver-manager", Image: renderData.Driver.ManagerImagePath}, + }, + }, + }, + }, + } + cl := fake.NewClientBuilder().WithScheme(sch).WithObjects(currentDs).Build() + sd.client = cl + + cr := newDriverCR("driver-a") + cr.Spec.Manager.Image = "" + + got, err := sd.handleDefaultImagesInObjects(context.Background(), desiredObjs, cr, *renderData) + require.NoError(t, err) + assert.Equal(t, desiredObjs, got) +} + +func TestHandleDefaultImagesCurrentGetError(t *testing.T) { + sch := driverTestScheme(t) + cl := fake.NewClientBuilder().WithScheme(sch). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, _ ...client.GetOption) error { + return fmt.Errorf("injected get error") + }, + }).Build() + sd := newTestStateDriver(t, cl, sch) + + cr := newDriverCR("driver-a") + cr.Spec.Manager.Image = "" + renderData := getMinimalDriverRenderData() + objs := []*unstructured.Unstructured{newDaemonSetUnstructured("ds-a", "test-operator")} + + _, err := sd.handleDefaultImagesInObjects(context.Background(), objs, cr, *renderData) + require.ErrorContains(t, err, "failed to get current driver DaemonSet") +} + +func TestHandleDefaultImagesReRenderError(t *testing.T) { + sch := driverTestScheme(t) + sd := newTestStateDriver(t, nil, sch) + + // Seed a current DaemonSet whose manager image differs from the render data's. + dsName := "nvidia-gpu-driver-ubuntu22.04" + currentDs := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: dsName, Namespace: "test-operator"}, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "k8s-driver-manager", Image: "old-manager:1.0"}}, + }, + }, + }, + } + cl := fake.NewClientBuilder().WithScheme(sch).WithObjects(currentDs).Build() + sd.client = cl + + cr := newDriverCR("driver-a") + cr.Spec.Manager.Image = "" + + // desiredObjs contains a valid DaemonSet (name/namespace match the seeded one), + // but the render data passed for re-render has a nil Driver.Spec, so the second + // render fails. + desiredObjs := []*unstructured.Unstructured{newDaemonSetUnstructured(dsName, "test-operator")} + renderData := &driverRenderData{ + Driver: &driverSpec{ManagerImagePath: "new-manager:2.0", Spec: nil}, + Runtime: &driverRuntimeSpec{Namespace: "test-operator"}, + } + + _, err := sd.handleDefaultImagesInObjects(context.Background(), desiredObjs, cr, *renderData) + require.ErrorContains(t, err, "failed to render kubernetes manifests") +} + +func TestHandleDefaultImagesReRenderSetRefError(t *testing.T) { + // Scheme without NVIDIADriver -> SetControllerReference on re-rendered DaemonSet fails. + sch := coreAppsScheme(t) + renderSd := newTestStateDriver(t, nil, driverTestScheme(t)) + renderData := getMinimalDriverRenderData() + renderData.Runtime.Namespace = "test-operator" + desiredObjs, err := renderSd.renderManifestObjects(context.Background(), renderData) + require.NoError(t, err) + desiredDs, err := getDaemonsetFromObjects(desiredObjs) + require.NoError(t, err) + + currentDs := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Name: desiredDs.Name, Namespace: desiredDs.Namespace}, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "k8s-driver-manager", Image: "old-manager:1.0"}}, + }, + }, + }, + } + cl := fake.NewClientBuilder().WithScheme(sch).WithObjects(currentDs).Build() + + sd := newTestStateDriver(t, cl, sch) + + cr := newDriverCR("driver-a") + cr.Spec.Manager.Image = "" + + _, err = sd.handleDefaultImagesInObjects(context.Background(), desiredObjs, cr, *renderData) + require.ErrorContains(t, err, "failed to set controller reference") +} + +func TestHandleDefaultImagesUnchangedSpecKeepsCurrentImage(t *testing.T) { + sch := driverTestScheme(t) + cr := newDriverCR("driver-a") + const currentImage = "custom-manager:1.0" + + // Replicate the production hashing steps to derive the hash the current + // DaemonSet must carry so that newHash == currentHash. + hashSd := newTestStateDriver(t, nil, sch) + hashData := getMinimalDriverRenderData() + hashData.Runtime.Namespace = "test-operator" + hashData.Driver.ManagerImagePath = currentImage + hashObjs, err := hashSd.renderManifestObjects(context.Background(), hashData) + require.NoError(t, err) + hashObj, err := getObjectOfKind(hashObjs, "DaemonSet") + require.NoError(t, err) + require.NoError(t, controllerutil.SetControllerReference(cr, hashObj, sch)) + hashSd.addStateSpecificLabels(hashObj) + expectedHash := utils.GetObjectHash(hashObj) + + // desiredObjs is rendered with the default manager image path (differs from currentImage). + renderSd := newTestStateDriver(t, nil, sch) + renderData := getMinimalDriverRenderData() + renderData.Runtime.Namespace = "test-operator" + desiredObjs, err := renderSd.renderManifestObjects(context.Background(), renderData) + require.NoError(t, err) + desiredDs, err := getDaemonsetFromObjects(desiredObjs) + require.NoError(t, err) + + currentDs := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: desiredDs.Name, + Namespace: desiredDs.Namespace, + Annotations: map[string]string{consts.NvidiaAnnotationHashKey: expectedHash}, + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{Name: "k8s-driver-manager", Image: currentImage}}, + }, + }, + }, + } + cl := fake.NewClientBuilder().WithScheme(sch).WithObjects(currentDs).Build() + + sd := newTestStateDriver(t, cl, sch) + + cr.Spec.Manager.Image = "" + got, err := sd.handleDefaultImagesInObjects(context.Background(), desiredObjs, cr, *renderData) + require.NoError(t, err) + // The returned objects use the current (unchanged) manager image. + gotDs, err := getDaemonsetFromObjects(got) + require.NoError(t, err) + assert.Equal(t, currentImage, managerImageFromDaemonSet(gotDs)) +} + +// --- buildDriverInstallConfig full field coverage ------------------------------ + +func TestBuildDriverInstallConfigAllFields(t *testing.T) { + data := &driverRenderData{ + Driver: &driverSpec{ + ImagePath: "nvcr.io/nvidia/driver:535-ubuntu22.04", + ManagerImagePath: "nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.6.2", + Spec: &nvidiav1alpha1.NVIDIADriverSpec{ + DriverType: nvidiav1alpha1.GPU, + KernelModuleType: "open", + Args: []string{"--foo"}, + SecretEnv: "secret-env", + Env: []nvidiav1alpha1.EnvVar{{Name: "A", Value: "1"}}, + Manager: nvidiav1alpha1.DriverManagerSpec{Env: []nvidiav1alpha1.EnvVar{{Name: "B", Value: "2"}}}, + LicensingConfig: &nvidiav1alpha1.DriverLicensingConfigSpec{SecretName: "lic-secret"}, + VirtualTopologyConfig: &nvidiav1alpha1.VirtualTopologyConfigSpec{Name: "topo"}, + KernelModuleConfig: &nvidiav1alpha1.KernelModuleConfigSpec{Name: "kmod"}, + RepoConfig: &nvidiav1alpha1.DriverRepoConfigSpec{Name: "repo"}, + CertConfig: &nvidiav1alpha1.DriverCertConfigSpec{Name: "cert"}, + }, + }, + GPUDirectRDMA: &nvidiav1alpha1.GPUDirectRDMASpec{ + Enabled: ptr.To(true), + UseHostMOFED: ptr.To(true), + }, + GDS: &gdsDriverSpec{ + ImagePath: "nvcr.io/nvidia/cloud-native/nvidia-fs:2.16.1", + Spec: &nvidiav1alpha1.GPUDirectStorageSpec{Enabled: ptr.To(true), Env: []nvidiav1alpha1.EnvVar{{Name: "G", Value: "1"}}}, + }, + GDRCopy: &gdrcopyDriverSpec{ + ImagePath: "nvcr.io/nvidia/cloud-native/gdrdrv:v2.4.1", + Spec: &nvidiav1alpha1.GDRCopySpec{Enabled: ptr.To(true), Env: []nvidiav1alpha1.EnvVar{{Name: "H", Value: "1"}}}, + }, + Runtime: &driverRuntimeSpec{ + Namespace: "test-operator", + OpenshiftVersion: "4.13", + OpenshiftDriverToolkitEnabled: true, + OpenshiftProxySpec: &configv1.ProxySpec{ + HTTPProxy: "http://proxy:8080", + HTTPSProxy: "https://proxy:8443", + NoProxy: "localhost", + TrustedCA: configv1.ConfigMapNameReference{Name: "trusted-ca"}, + }, + }, + Openshift: &openshiftSpec{ + ToolkitImage: "quay.io/toolkit:latest", + RHCOSVersion: "413.92", + }, + Precompiled: &precompiledSpec{ + KernelVersion: "5.15.0-70-generic", + }, + AdditionalConfigs: &additionalConfigs{ + VolumeMounts: []corev1.VolumeMount{{Name: "vm", MountPath: "/x"}}, + Volumes: []corev1.Volume{{Name: "vm"}}, + }, + HostRoot: "/host", + } + + config := buildDriverInstallConfig(data) + require.NotNil(t, config) + + // Compare the entire mapped install config in one shot so every field + // buildDriverInstallConfig populates is covered by the assertion. + want := driverconfig.DriverInstallState{ + DriverImage: "nvcr.io/nvidia/driver:535-ubuntu22.04", + DriverManagerImage: "nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.6.2", + PeermemImage: "nvcr.io/nvidia/driver:535-ubuntu22.04", + GDSImage: "nvcr.io/nvidia/cloud-native/nvidia-fs:2.16.1", + GDRCopyImage: "nvcr.io/nvidia/cloud-native/gdrdrv:v2.4.1", + DTKImage: "quay.io/toolkit:latest", + DriverType: "gpu", + KernelModuleType: "open", + DriverArgs: []string{"--foo"}, + DriverEnv: []driverconfig.EnvVar{{Name: "A", Value: "1"}}, + ManagerEnv: []driverconfig.EnvVar{{Name: "B", Value: "2"}}, + GDSEnv: []driverconfig.EnvVar{{Name: "G", Value: "1"}}, + GDRCopyEnv: []driverconfig.EnvVar{{Name: "H", Value: "1"}}, + SecretEnvSource: "secret-env", + GPUDirectRDMAEnabled: true, + UseHostMOFED: true, + GDSEnabled: true, + GDRCopyEnabled: true, + LicensingConfigName: "lic-secret", + VirtualTopologyConfig: "topo", + KernelModuleConfig: "kmod", + RepoConfig: "repo", + CertConfig: "cert", + UsePrecompiled: true, + KernelVersion: "5.15.0-70-generic", + OpenshiftVersion: "4.13", + DTKEnabled: true, + RHCOSVersion: "413.92", + HTTPProxy: "http://proxy:8080", + HTTPSProxy: "https://proxy:8443", + NoProxy: "localhost", + TrustedCAConfigMapName: "trusted-ca", + AdditionalVolumes: []driverconfig.VolumeConfig{{Name: "vm"}}, + AdditionalVolumeMounts: []driverconfig.VolumeMountConfig{{Name: "vm", MountPath: "/x"}}, + HostRoot: "/host", + } + + diff := cmp.Diff(want, *config, cmpopts.EquateEmpty()) + assert.Empty(t, diff, "unexpected driver install config (-want +got):\n%s", diff) +} + +// --- GetWatchSources (driver.go) ----------------------------------------------- + +// fakeManager implements just enough of ctrl.Manager for GetWatchSources. +type fakeManager struct { + ctrl.Manager + cache cache.Cache + scheme *runtime.Scheme + mapper meta.RESTMapper +} + +func (f *fakeManager) GetCache() cache.Cache { return f.cache } +func (f *fakeManager) GetScheme() *runtime.Scheme { return f.scheme } +func (f *fakeManager) GetRESTMapper() meta.RESTMapper { return f.mapper } + +// --- getNodePools list error --------------------------------------------------- + +func TestGetNodePoolsListError(t *testing.T) { + sch := driverTestScheme(t) + cl := fake.NewClientBuilder().WithScheme(sch). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(_ context.Context, _ client.WithWatch, _ client.ObjectList, _ ...client.ListOption) error { + return fmt.Errorf("injected node list error") + }, + }).Build() + + cr := &nvidiav1alpha1.NVIDIADriver{ObjectMeta: metav1.ObjectMeta{Name: "driver-a"}} + _, err := getNodePools(context.Background(), cl, cr, false) + require.ErrorContains(t, err, "injected node list error") +} + +func TestDriverGetWatchSources(t *testing.T) { + sch := driverTestScheme(t) + + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{ + {Group: "nvidia.com", Version: "v1alpha1"}, + }) + mapper.Add(schema.GroupVersionKind{Group: "nvidia.com", Version: "v1alpha1", Kind: "NVIDIADriver"}, meta.RESTScopeRoot) + + sd := newTestStateDriver(t, nil, sch) + + mgr := &fakeManager{scheme: sch, mapper: mapper} + sources := sd.GetWatchSources(mgr) + require.Contains(t, sources, "DaemonSet") + assert.NotNil(t, sources["DaemonSet"]) +} diff --git a/internal/state/driver_sync_test.go b/internal/state/driver_sync_test.go new file mode 100644 index 0000000000..a75fbb7abe --- /dev/null +++ b/internal/state/driver_sync_test.go @@ -0,0 +1,457 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package state + +import ( + "context" + "fmt" + "strings" + "testing" + + configv1 "github.com/openshift/api/config/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + apitypes "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +// fakeClusterInfo is a configurable clusterinfo.Interface implementation. +type fakeClusterInfo struct { + runtime string + runtimeErr error + openshiftVersion string + openshiftVersionErr error + dtkImages map[string]string + proxySpec *configv1.ProxySpec + proxyErr error +} + +func (f fakeClusterInfo) GetContainerRuntime() (string, error) { + return f.runtime, f.runtimeErr +} + +func (f fakeClusterInfo) GetOpenshiftVersion() (string, error) { + return f.openshiftVersion, f.openshiftVersionErr +} + +func (f fakeClusterInfo) GetOpenshiftDriverToolkitImages() map[string]string { + return f.dtkImages +} + +func (f fakeClusterInfo) GetOpenshiftProxySpec() (*configv1.ProxySpec, error) { + return f.proxySpec, f.proxyErr +} + +func (f fakeClusterInfo) GetDRAResourceGVR() (schema.GroupVersionResource, bool, error) { + return schema.GroupVersionResource{}, false, nil +} + +func driverTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(s)) + require.NoError(t, appsv1.AddToScheme(s)) + require.NoError(t, rbacv1.AddToScheme(s)) + require.NoError(t, nvidiav1alpha1.AddToScheme(s)) + return s +} + +// newTestStateDriver builds a *stateDriver, failing the test if construction or +// the type assertion fails instead of panicking on a discarded error. +func newTestStateDriver(t *testing.T, cl client.Client, sch *runtime.Scheme) *stateDriver { + t.Helper() + state, err := NewStateDriver(cl, "test-operator", sch, manifestDir) + require.NoError(t, err) + sd, ok := state.(*stateDriver) + require.True(t, ok) + return sd +} + +func TestGetGDSSpec(t *testing.T) { + pool := nodePool{osTag: "ubuntu22.04"} + enabledSpec := func(image string) *nvidiav1alpha1.NVIDIADriverSpec { + return &nvidiav1alpha1.NVIDIADriverSpec{ + GPUDirectStorage: &nvidiav1alpha1.GPUDirectStorageSpec{ + Enabled: ptr.To(true), Repository: "nvcr.io/nvidia/cloud-native", Image: image, Version: "2.16.1", + }, + } + } + testCases := []struct { + name string + spec *nvidiav1alpha1.NVIDIADriverSpec + wantNil bool + wantErr bool + wantImage string + }{ + {name: "nil spec", spec: nil, wantNil: true}, + {name: "disabled", spec: &nvidiav1alpha1.NVIDIADriverSpec{}, wantNil: true}, + {name: "enabled resolves image path", spec: enabledSpec("nvidia-fs"), wantImage: "nvcr.io/nvidia/cloud-native/nvidia-fs:2.16.1-ubuntu22.04"}, + {name: "enabled with invalid image errors", spec: enabledSpec("INVALID IMAGE"), wantErr: true}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gds, err := getGDSSpec(tc.spec, pool) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + if tc.wantNil { + assert.Nil(t, gds) + return + } + require.NotNil(t, gds) + assert.Equal(t, tc.wantImage, gds.ImagePath) + }) + } +} + +func TestGetGDRCopySpec(t *testing.T) { + pool := nodePool{osTag: "ubuntu22.04"} + enabledSpec := func(image string) *nvidiav1alpha1.NVIDIADriverSpec { + return &nvidiav1alpha1.NVIDIADriverSpec{ + GDRCopy: &nvidiav1alpha1.GDRCopySpec{ + Enabled: ptr.To(true), Repository: "nvcr.io/nvidia/cloud-native", Image: image, Version: "v2.4.1", + }, + } + } + testCases := []struct { + name string + spec *nvidiav1alpha1.NVIDIADriverSpec + wantNil bool + wantErr bool + wantImage string + }{ + {name: "nil spec", spec: nil, wantNil: true}, + {name: "disabled", spec: &nvidiav1alpha1.NVIDIADriverSpec{}, wantNil: true}, + {name: "enabled resolves image path", spec: enabledSpec("gdrdrv"), wantImage: "nvcr.io/nvidia/cloud-native/gdrdrv:v2.4.1-ubuntu22.04"}, + {name: "enabled with invalid image errors", spec: enabledSpec("INVALID IMAGE"), wantErr: true}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gdr, err := getGDRCopySpec(tc.spec, pool) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + if tc.wantNil { + assert.Nil(t, gdr) + return + } + require.NotNil(t, gdr) + assert.Equal(t, tc.wantImage, gdr.ImagePath) + }) + } +} + +func TestGetRuntimeSpec(t *testing.T) { + spec := &nvidiav1alpha1.NVIDIADriverSpec{} + + t.Run("non-openshift", func(t *testing.T) { + info := fakeClusterInfo{openshiftVersion: ""} + rs, err := getRuntimeSpec("test-ns", info, spec) + require.NoError(t, err) + assert.Equal(t, "test-ns", rs.Namespace) + assert.Empty(t, rs.OpenshiftVersion) + assert.False(t, rs.OpenshiftDriverToolkitEnabled) + }) + + t.Run("openshift version error", func(t *testing.T) { + info := fakeClusterInfo{openshiftVersionErr: fmt.Errorf("boom")} + _, err := getRuntimeSpec("test-ns", info, spec) + require.ErrorContains(t, err, "failed to get openshift version") + }) + + t.Run("openshift with DTK enabled", func(t *testing.T) { + info := fakeClusterInfo{ + openshiftVersion: "4.13", + dtkImages: map[string]string{"413.92": "some-image"}, + proxySpec: &configv1.ProxySpec{HTTPProxy: "http://proxy:8080"}, + } + rs, err := getRuntimeSpec("test-ns", info, spec) + require.NoError(t, err) + assert.Equal(t, "4.13", rs.OpenshiftVersion) + assert.True(t, rs.OpenshiftDriverToolkitEnabled) + require.NotNil(t, rs.OpenshiftProxySpec) + assert.Equal(t, "http://proxy:8080", rs.OpenshiftProxySpec.HTTPProxy) + }) + + t.Run("openshift proxy error", func(t *testing.T) { + info := fakeClusterInfo{ + openshiftVersion: "4.13", + proxyErr: fmt.Errorf("proxy boom"), + } + _, err := getRuntimeSpec("test-ns", info, spec) + require.ErrorContains(t, err, "failed to retrieve proxy settings") + }) + + t.Run("openshift with precompiled skips DTK", func(t *testing.T) { + precompiledSpec := &nvidiav1alpha1.NVIDIADriverSpec{UsePrecompiled: ptr.To(true)} + info := fakeClusterInfo{ + openshiftVersion: "4.13", + dtkImages: map[string]string{"413.92": "some-image"}, + } + rs, err := getRuntimeSpec("test-ns", info, precompiledSpec) + require.NoError(t, err) + assert.Equal(t, "4.13", rs.OpenshiftVersion) + assert.False(t, rs.OpenshiftDriverToolkitEnabled) + }) +} + +func TestRenderManifestObjects(t *testing.T) { + state, err := NewStateDriver(nil, "", nil, manifestDir) + require.NoError(t, err) + sd := state.(*stateDriver) + + objs, err := sd.renderManifestObjects(context.Background(), getMinimalDriverRenderData()) + require.NoError(t, err) + require.NotEmpty(t, objs) +} + +func newGPUNode(name, owner string) *corev1.Node { + return newGPUNodeOS(name, owner, "ubuntu", "22.04") +} + +func newDriverCR(name string) *nvidiav1alpha1.NVIDIADriver { + return &nvidiav1alpha1.NVIDIADriver{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: apitypes.UID("test-uid-" + name), + }, + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + DriverType: nvidiav1alpha1.GPU, + Repository: "nvcr.io/nvidia", + Image: "driver", + Version: "535.104.05", + Manager: nvidiav1alpha1.DriverManagerSpec{ + Repository: "nvcr.io/nvidia/cloud-native", + Image: "k8s-driver-manager", + Version: "v0.6.2", + }, + }, + } +} + +func driverIndexBuilder(sch *runtime.Scheme, objs ...client.Object) client.Client { + return fake.NewClientBuilder(). + WithScheme(sch). + WithObjects(objs...). + WithIndex(&appsv1.DaemonSet{}, consts.NVIDIADriverControllerIndexKey, func(_ client.Object) []string { + return nil + }). + Build() +} + +func TestGetManifestObjectsMissingCatalogEntries(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch) + sd := newTestStateDriver(t, cl, sch) + cr := newDriverCR("driver-a") + + // Missing host root. + _, err := sd.getManifestObjects(context.Background(), cr, NewInfoCatalog()) + require.ErrorContains(t, err, "failed to get host root from info catalog") + + // Host root present, but ClusterInfo missing. + catalog := NewInfoCatalog() + catalog.Add(InfoTypeHostRoot, "/host") + _, err = sd.getManifestObjects(context.Background(), cr, catalog) + require.ErrorContains(t, err, "failed to get cluster info") +} + +func TestGetManifestObjectsNoNodes(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch) + sd := newTestStateDriver(t, cl, sch) + cr := newDriverCR("driver-a") + + catalog := NewInfoCatalog() + catalog.Add(InfoTypeHostRoot, "/host") + catalog.Add(InfoTypeClusterInfo, fakeClusterInfo{}) + + objs, err := sd.getManifestObjects(context.Background(), cr, catalog) + require.NoError(t, err) + assert.Empty(t, objs) +} + +func TestGetManifestObjectsWithNode(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch, newGPUNode("gpu-node", "driver-a")) + sd := newTestStateDriver(t, cl, sch) + cr := newDriverCR("driver-a") + + catalog := NewInfoCatalog() + catalog.Add(InfoTypeHostRoot, "/host") + catalog.Add(InfoTypeClusterInfo, fakeClusterInfo{}) + + objs, err := sd.getManifestObjects(context.Background(), cr, catalog) + require.NoError(t, err) + require.NotEmpty(t, objs) + + // A DaemonSet should be among the rendered objects. + _, err = getObjectOfKind(objs, "DaemonSet") + require.NoError(t, err) +} + +func TestSyncWrongCRType(t *testing.T) { + state, err := NewStateDriver(nil, "", nil, manifestDir) + require.NoError(t, err) + sd := state.(*stateDriver) + + syncState, err := sd.Sync(context.Background(), "not-a-cr", NewInfoCatalog()) + require.Error(t, err) + assert.Equal(t, SyncState(SyncStateError), syncState) +} + +func TestSyncNoNodesReady(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch) + sd := newTestStateDriver(t, cl, sch) + cr := newDriverCR("driver-a") + + catalog := NewInfoCatalog() + catalog.Add(InfoTypeHostRoot, "/host") + catalog.Add(InfoTypeClusterInfo, fakeClusterInfo{}) + + // No nodes -> no objects -> sync reports ready. + syncState, err := sd.Sync(context.Background(), cr, catalog) + require.NoError(t, err) + assert.Equal(t, SyncState(SyncStateReady), syncState) +} + +func TestSyncCreatesObjects(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch, newGPUNode("gpu-node", "driver-a")) + sd := newTestStateDriver(t, cl, sch) + cr := newDriverCR("driver-a") + + catalog := NewInfoCatalog() + catalog.Add(InfoTypeHostRoot, "/host") + catalog.Add(InfoTypeClusterInfo, fakeClusterInfo{}) + + // Sync creates the driver objects without error. Readiness is intentionally + // not asserted here: the fake client runs no DaemonSet controller, so the + // reported state would only reflect a zero-valued status. Readiness is + // covered directly with explicit statuses in TestIsDaemonSetReady. + _, err := sd.Sync(context.Background(), cr, catalog) + require.NoError(t, err) + + // Verify the DaemonSet was created and carries the render + apply metadata: + // correct name/namespace, controller owner reference, state label, and hash. + dsList := &appsv1.DaemonSetList{} + require.NoError(t, cl.List(context.Background(), dsList)) + require.Len(t, dsList.Items, 1) + ds := dsList.Items[0] + + assert.True(t, strings.HasPrefix(ds.Name, "nvidia-gpu-driver-ubuntu22.04-"), "unexpected DaemonSet name %q", ds.Name) + assert.Equal(t, "test-operator", ds.Namespace) + assert.Equal(t, "state-driver", ds.Labels[consts.StateLabel]) + assert.NotEmpty(t, ds.Annotations[consts.NvidiaAnnotationHashKey]) + + require.Len(t, ds.OwnerReferences, 1) + owner := ds.OwnerReferences[0] + assert.Equal(t, "NVIDIADriver", owner.Kind) + assert.Equal(t, cr.Name, owner.Name) + assert.Equal(t, cr.UID, owner.UID) + require.NotNil(t, owner.Controller) + assert.True(t, *owner.Controller) +} + +func TestGetDriverName(t *testing.T) { + testCases := []struct { + name string + driverType nvidiav1alpha1.DriverType + want string + }{ + {"GPU", nvidiav1alpha1.GPU, "nvidia-gpu-driver-my-driver-ubuntu22.04"}, + {"vGPU host manager", nvidiav1alpha1.VGPUHostManager, "nvidia-vgpu-manager-my-driver-ubuntu22.04"}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cr := &nvidiav1alpha1.NVIDIADriver{ + ObjectMeta: metav1.ObjectMeta{Name: "my-driver"}, + Spec: nvidiav1alpha1.NVIDIADriverSpec{DriverType: tc.driverType}, + } + assert.Equal(t, tc.want, getDriverName(cr, "ubuntu22.04")) + }) + } +} + +func TestGetDriverSpecErrors(t *testing.T) { + // nil CR -> error. + _, err := getDriverSpec(nil, nodePool{}) + require.ErrorContains(t, err, "no NVIDIADriver CR provided") + + // Invalid driver image reference -> error. + badImageCR := &nvidiav1alpha1.NVIDIADriver{ + ObjectMeta: metav1.ObjectMeta{Name: "driver-a"}, + Spec: nvidiav1alpha1.NVIDIADriverSpec{ + DriverType: nvidiav1alpha1.GPU, + Repository: "nvcr.io/nvidia", + Image: "INVALID IMAGE", + Version: "535.104.05", + }, + } + _, err = getDriverSpec(badImageCR, nodePool{osTag: "ubuntu22.04"}) + require.Error(t, err) +} + +func TestHandleDefaultImagesInObjectsManagerImageSet(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch) + sd := newTestStateDriver(t, cl, sch) + + // Manager.Image is set, so the default-image handling returns the objects unchanged. + cr := newDriverCR("driver-a") + renderData := getMinimalDriverRenderData() + objs := []*unstructured.Unstructured{newDaemonSetUnstructured("ds-a", "test-operator")} + + got, err := sd.handleDefaultImagesInObjects(context.Background(), objs, cr, *renderData) + require.NoError(t, err) + assert.Equal(t, objs, got) +} + +func TestHandleDefaultImagesInObjectsCurrentDaemonSetNotFound(t *testing.T) { + sch := driverTestScheme(t) + cl := driverIndexBuilder(sch) + sd := newTestStateDriver(t, cl, sch) + + // Manager.Image empty -> env var path; current DaemonSet does not exist -> returns objs unchanged. + cr := newDriverCR("driver-a") + cr.Spec.Manager.Image = "" + renderData := getMinimalDriverRenderData() + + daemonSet := newDaemonSetUnstructured("nvidia-gpu-driver-test", "test-operator") + objs := []*unstructured.Unstructured{daemonSet} + + got, err := sd.handleDefaultImagesInObjects(context.Background(), objs, cr, *renderData) + require.NoError(t, err) + assert.Equal(t, objs, got) +} diff --git a/internal/state/info_source_test.go b/internal/state/info_source_test.go new file mode 100644 index 0000000000..36efab57d8 --- /dev/null +++ b/internal/state/info_source_test.go @@ -0,0 +1,45 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package state + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInfoCatalogAddAndGet(t *testing.T) { + catalog := NewInfoCatalog() + require.NotNil(t, catalog) + + // Getting an entry that has not been added returns nil. + assert.Nil(t, catalog.Get(InfoTypeClusterInfo)) + + clusterInfo := "some-cluster-info" + policy := struct{ Name string }{Name: "policy"} + + catalog.Add(InfoTypeClusterInfo, clusterInfo) + catalog.Add(InfoTypeHostRoot, policy) + + assert.Equal(t, clusterInfo, catalog.Get(InfoTypeClusterInfo)) + assert.Equal(t, policy, catalog.Get(InfoTypeHostRoot)) + + // Overwriting an existing entry replaces the value. + catalog.Add(InfoTypeClusterInfo, "updated") + assert.Equal(t, "updated", catalog.Get(InfoTypeClusterInfo)) +} diff --git a/internal/state/manager_test.go b/internal/state/manager_test.go new file mode 100644 index 0000000000..0445fcea7f --- /dev/null +++ b/internal/state/manager_test.go @@ -0,0 +1,216 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package state + +import ( + "context" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" +) + +// requireMissingManifestDir skips the test when the given hardcoded manifest +// directory exists. newStates/NewManager load manifests from fixed paths with no +// injection point, so the error these tests assert only occurs when the directory +// is absent (true in CI/dev). Skipping keeps the tests from failing inside an +// operator image where the manifests are installed. +func requireMissingManifestDir(t *testing.T, dir string) { + t.Helper() + if _, err := os.Stat(dir); err == nil { + t.Skipf("manifests installed at %s; the factory would not fail here", dir) + } +} + +// fakeState is a minimal State implementation used to drive the stateManager. +type fakeState struct { + name string + description string + syncState SyncState + syncErr error + watchSources map[string]SyncingSource +} + +func (f *fakeState) Name() string { return f.name } +func (f *fakeState) Description() string { return f.description } +func (f *fakeState) Sync(_ context.Context, _ interface{}, _ InfoCatalog) (SyncState, error) { + return f.syncState, f.syncErr +} +func (f *fakeState) GetWatchSources(_ ctrlManager) map[string]SyncingSource { + return f.watchSources +} + +// fakeSyncingSource is a comparable SyncingSource so tests can assert which +// concrete source instance survives watch-source deduplication. +type fakeSyncingSource struct{ id string } + +func (fakeSyncingSource) Start(context.Context, workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + return nil +} +func (fakeSyncingSource) WaitForSync(context.Context) error { return nil } + +func TestSyncState(t *testing.T) { + testCases := []struct { + description string + states []*fakeState + expectedStatus SyncState + expectErrInfo bool + }{ + { + description: "all states ready aggregates to ready", + states: []*fakeState{ + {name: "state-a", syncState: SyncStateReady}, + {name: "state-b", syncState: SyncStateReady}, + }, + expectedStatus: SyncStateReady, + }, + { + description: "any not-ready state aggregates to not ready", + states: []*fakeState{ + {name: "state-a", syncState: SyncStateReady}, + {name: "state-b", syncState: SyncStateNotReady}, + }, + expectedStatus: SyncStateNotReady, + }, + { + description: "an errored state aggregates to not ready and records the error", + states: []*fakeState{ + {name: "state-a", syncState: SyncStateError, syncErr: fmt.Errorf("boom")}, + }, + expectedStatus: SyncStateNotReady, + expectErrInfo: true, + }, + { + // Only NotReady and Error hold back readiness, so Ignore (returned by a + // state after its objects are deleted) still aggregates to ready. + description: "ignored states aggregate to ready", + states: []*fakeState{ + {name: "state-a", syncState: SyncStateIgnore}, + {name: "state-b", syncState: SyncStateReady}, + }, + expectedStatus: SyncStateReady, + }, + { + // No states to hold back readiness, so the aggregate is ready. + description: "an empty state list aggregates to ready", + states: nil, + expectedStatus: SyncStateReady, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + states := make([]State, len(tc.states)) + for i := range tc.states { + states[i] = tc.states[i] + } + mgr := &stateManager{states: states} + + res := mgr.SyncState(context.Background(), nil, NewInfoCatalog()) + + assert.Equal(t, tc.expectedStatus, res.Status) + require.Len(t, res.StatesStatus, len(tc.states)) + // Each per-state result must reflect that state's name and status. + for i, s := range tc.states { + assert.Equal(t, s.name, res.StatesStatus[i].StateName) + assert.Equal(t, s.syncState, res.StatesStatus[i].Status) + } + if tc.expectErrInfo { + assert.Error(t, res.StatesStatus[0].ErrInfo) + } + }) + } +} + +func TestGetWatchSourcesDeduplicates(t *testing.T) { + dsFromA := fakeSyncingSource{id: "ds-from-a"} + dsFromB := fakeSyncingSource{id: "ds-from-b"} + cmFromB := fakeSyncingSource{id: "cm-from-b"} + + mgr := &stateManager{ + states: []State{ + &fakeState{name: "state-a", watchSources: map[string]SyncingSource{"DaemonSet": dsFromA}}, + // state-b re-advertises "DaemonSet"; for a duplicate key the first state wins. + &fakeState{name: "state-b", watchSources: map[string]SyncingSource{"DaemonSet": dsFromB, "ConfigMap": cmFromB}}, + }, + } + + sources := mgr.GetWatchSources(nil) + + // The result order is nondeterministic (collected from a map), so compare as a set. + got := make(map[SyncingSource]bool, len(sources)) + for _, s := range sources { + got[s] = true + } + assert.Len(t, sources, 2) + assert.True(t, got[dsFromA], "first state's DaemonSet source should be retained") + assert.False(t, got[dsFromB], "second state's duplicate DaemonSet source should be dropped") + assert.True(t, got[cmFromB], "unique ConfigMap source should be retained") +} + +func TestNewManagerUnsupportedCRD(t *testing.T) { + mgr, err := NewManager("UnsupportedKind", "test-ns", nil, runtime.NewScheme()) + require.Error(t, err) + assert.Nil(t, mgr) + assert.Contains(t, err.Error(), "failed to add states") +} + +func TestNewStatesUnsupportedCRD(t *testing.T) { + states, err := newStates("UnsupportedKind", "test-ns", nil, runtime.NewScheme()) + require.Error(t, err) + assert.Nil(t, states) + assert.Contains(t, err.Error(), "unsupported CRD") +} + +// TestNewStatesNVIDIADriverCase exercises the NVIDIADriver branch of newStates +// and newNVIDIADriverStates. NewStateDriver fails because the hardcoded manifest +// directory does not exist, so the error is propagated. +func TestNewStatesNVIDIADriverCase(t *testing.T) { + requireMissingManifestDir(t, "/opt/gpu-operator/manifests/state-driver") + states, err := newStates(nvidiav1alpha1.NVIDIADriverCRDName, "test-ns", nil, runtime.NewScheme()) + require.Error(t, err) + assert.Nil(t, states) + assert.Contains(t, err.Error(), "failed to create NVIDIA driver state") +} + +// TestNewStatesGPUClusterCase exercises the GPUCluster dispatch branch of newStates +// and newGPUClusterStates. The first operand (DRA driver) fails on its missing +// hardcoded manifest directory, so the error is propagated. +func TestNewStatesGPUClusterCase(t *testing.T) { + requireMissingManifestDir(t, "/opt/gpu-operator/manifests/state-dra-driver") + states, err := newStates(nvidiav1alpha1.GPUClusterCRDName, "test-ns", nil, runtime.NewScheme()) + require.Error(t, err) + assert.Nil(t, states) + assert.Contains(t, err.Error(), "failed to create DRA driver state") +} + +// TestNewManagerNVIDIADriverCase drives NewManager through the NVIDIADriver +// state factory (which fails on the missing manifest directory). +func TestNewManagerNVIDIADriverCase(t *testing.T) { + requireMissingManifestDir(t, "/opt/gpu-operator/manifests/state-driver") + mgr, err := NewManager(nvidiav1alpha1.NVIDIADriverCRDName, "test-ns", nil, runtime.NewScheme()) + require.Error(t, err) + assert.Nil(t, mgr) + assert.Contains(t, err.Error(), "failed to add states") +} diff --git a/internal/state/state_skel_reconcile_test.go b/internal/state/state_skel_reconcile_test.go new file mode 100644 index 0000000000..0c4e80e1bf --- /dev/null +++ b/internal/state/state_skel_reconcile_test.go @@ -0,0 +1,487 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package state + +import ( + "context" + "fmt" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "github.com/NVIDIA/gpu-operator/internal/consts" +) + +// TestCreateOrUpdateObjsGetError covers the branch where an object already +// exists but the subsequent Get fails. +func TestCreateOrUpdateObjsGetError(t *testing.T) { + existing := newConfigMapUnstructured("cm-a", "test-ns") + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(existing). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, _ ...client.GetOption) error { + return fmt.Errorf("injected get error") + }, + }).Build() + s := newTestSkel(t, cl) + + desired := newConfigMapUnstructured("cm-a", "test-ns") + noop := func(_ *unstructured.Unstructured) error { return nil } + err := s.createOrUpdateObjs(context.Background(), noop, []*unstructured.Unstructured{desired}) + require.ErrorContains(t, err, "injected get error") +} + +// TestCreateOrUpdateObjsMergeError covers the mergeObjects error branch: an +// existing ServiceAccount whose "secrets" field is malformed makes +// mergeServiceAccount fail. +func TestCreateOrUpdateObjsMergeError(t *testing.T) { + existing := newServiceAccountUnstructured("sa-a", "test-ns") + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(existing). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, obj client.Object, _ ...client.GetOption) error { + // Return a ServiceAccount whose secrets field is not a slice. + u, ok := obj.(*unstructured.Unstructured) + if !ok { + return fmt.Errorf("unexpected object type") + } + u.SetGroupVersionKind(schema.GroupVersionKind{Version: "v1", Kind: "ServiceAccount"}) + u.SetName("sa-a") + u.SetNamespace("test-ns") + u.Object["secrets"] = "not-a-slice" + return nil + }, + }).Build() + s := newTestSkel(t, cl) + + desired := newServiceAccountUnstructured("sa-a", "test-ns") + noop := func(_ *unstructured.Unstructured) error { return nil } + err := s.createOrUpdateObjs(context.Background(), noop, []*unstructured.Unstructured{desired}) + require.Error(t, err) +} + +// TestCreateOrUpdateObjsUpdateError covers the updateObj error branch during +// create-or-update of an existing object. +func TestCreateOrUpdateObjsUpdateError(t *testing.T) { + existing := newConfigMapUnstructured("cm-a", "test-ns") + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(existing). + WithInterceptorFuncs(interceptor.Funcs{ + Update: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.UpdateOption) error { + return fmt.Errorf("injected update error") + }, + }).Build() + s := newTestSkel(t, cl) + + desired := newConfigMapUnstructured("cm-a", "test-ns") + require.NoError(t, unstructured.SetNestedStringMap(desired.Object, map[string]string{"key": "new"}, "data")) + noop := func(_ *unstructured.Unstructured) error { return nil } + err := s.createOrUpdateObjs(context.Background(), noop, []*unstructured.Unstructured{desired}) + require.ErrorContains(t, err, "failed to update resource") +} + +// TestMergeServiceAccountErrors covers the NestedSlice error branches for both +// secrets and imagePullSecrets fields. +func TestMergeServiceAccountErrors(t *testing.T) { + s := newTestSkel(t, fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build()) + + t.Run("malformed secrets", func(t *testing.T) { + updated := newServiceAccountUnstructured("sa", "test-ns") + current := newServiceAccountUnstructured("sa", "test-ns") + current.Object["secrets"] = "not-a-slice" + err := s.mergeServiceAccount(updated, current) + require.Error(t, err) + }) + + t.Run("malformed imagePullSecrets", func(t *testing.T) { + updated := newServiceAccountUnstructured("sa", "test-ns") + current := newServiceAccountUnstructured("sa", "test-ns") + require.NoError(t, unstructured.SetNestedSlice(current.Object, + []interface{}{map[string]interface{}{"name": "s"}}, "secrets")) + current.Object["imagePullSecrets"] = "not-a-slice" + err := s.mergeServiceAccount(updated, current) + require.Error(t, err) + }) +} + +// TestIsDaemonSetReadyErrors covers the JSON marshal and unmarshal error paths. +func TestIsDaemonSetReadyErrors(t *testing.T) { + s := newTestSkel(t, fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build()) + + t.Run("marshal error", func(t *testing.T) { + daemonSet := newDaemonSetUnstructured("ds", "test-ns") + // A channel value cannot be marshalled to JSON. + daemonSet.Object["bad"] = make(chan int) + _, err := s.isDaemonSetReady(daemonSet, logr.Discard()) + require.ErrorContains(t, err, "failed to marshall unstructured daemonset object") + }) + + t.Run("unmarshal error", func(t *testing.T) { + daemonSet := newDaemonSetUnstructured("ds", "test-ns") + // status must be an object; a string marshals fine but fails to unmarshal + // into the typed DaemonSet.Status struct. + daemonSet.Object["status"] = "not-an-object" + _, err := s.isDaemonSetReady(daemonSet, logr.Discard()) + require.ErrorContains(t, err, "failed to unmarshall to daemonset object") + }) +} + +// TestGetObjNotFoundHelper verifies IsNotFound classification on a missing object. +func TestGetObjNotFound(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build() + s := newTestSkel(t, cl) + missing := newConfigMapUnstructured("missing", "test-ns") + err := s.getObj(context.Background(), missing) + require.True(t, apierrors.IsNotFound(err)) +} + +// TestCreateObjAlreadyExists verifies the AlreadyExists branch in createObj. +func TestCreateObjAlreadyExists(t *testing.T) { + obj := newConfigMapUnstructured("cm", "test-ns") + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(obj).Build() + s := newTestSkel(t, cl) + + err := s.createObj(context.Background(), newConfigMapUnstructured("cm", "test-ns")) + require.True(t, apierrors.IsAlreadyExists(err)) +} + +// --- readiness predicates ------------------------------------------------------ + +// TestIsDaemonSetReadyStaleGeneration characterizes current behavior: the +// nonzero-desired-pods path of isDaemonSetReady does NOT re-check +// ObservedGeneration, so a DaemonSet whose status is still from a previous +// generation (Generation=2, ObservedGeneration=1) is reported ready as soon as +// the replica counts line up. Kubernetes' rollout-status gates on +// ObservedGeneration first; this predicate does not. Update this test if the +// generation check is extended to the nonzero-desired path. +func TestIsDaemonSetReadyStaleGeneration(t *testing.T) { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Generation: 2}, + Status: appsv1.DaemonSetStatus{ + ObservedGeneration: 1, + DesiredNumberScheduled: 2, + CurrentNumberScheduled: 2, + NumberAvailable: 2, + UpdatedNumberScheduled: 2, + }, + } + s := &stateSkel{} + ready, err := s.isDaemonSetReady(toUnstructuredDaemonSet(t, ds), logr.Discard()) + require.NoError(t, err) + assert.True(t, ready, "known gap: stale-generation status is currently treated as ready") +} + +func toUnstructuredDeployment(t *testing.T, dep *appsv1.Deployment) *unstructured.Unstructured { + t.Helper() + obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(dep) + require.NoError(t, err) + return &unstructured.Unstructured{Object: obj} +} + +func TestIsDeploymentReady(t *testing.T) { + testCases := []struct { + name string + dep *appsv1.Deployment + expected bool + }{ + { + name: "all counts match and generation observed", + dep: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](1)}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 1, Replicas: 1, UpdatedReplicas: 1, AvailableReplicas: 1}, + }, + expected: true, + }, + { + name: "nil replicas defaults to one", + dep: &appsv1.Deployment{ + Status: appsv1.DeploymentStatus{Replicas: 1, UpdatedReplicas: 1, AvailableReplicas: 1}, + }, + expected: true, + }, + { + name: "zero desired replicas", + dep: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](0)}, + }, + expected: true, + }, + { + name: "generation not yet observed", + dep: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Generation: 2}, + Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](1)}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 1, UpdatedReplicas: 1, AvailableReplicas: 1}, + }, + expected: false, + }, + { + name: "updated replicas below desired", + dep: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](2)}, + Status: appsv1.DeploymentStatus{UpdatedReplicas: 1, AvailableReplicas: 2}, + }, + expected: false, + }, + { + name: "available replicas below desired", + dep: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](2)}, + Status: appsv1.DeploymentStatus{UpdatedReplicas: 2, AvailableReplicas: 1}, + }, + expected: false, + }, + { + // Characterizes current behavior: isDeploymentReady does not require + // Status.Replicas == desired, so a rollout with a superseded replica + // still terminating (Replicas=2, desired=1) is reported ready. + // Kubernetes' DeploymentComplete requires Replicas == desired for this + // reason. Update this case if the total-replicas check is added. + name: "old replica still terminating is currently ready", + dep: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](1)}, + Status: appsv1.DeploymentStatus{Replicas: 2, UpdatedReplicas: 1, AvailableReplicas: 1}, + }, + expected: true, + }, + } + + s := &stateSkel{} + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ready, err := s.isDeploymentReady(toUnstructuredDeployment(t, tc.dep), logr.Discard()) + require.NoError(t, err) + assert.Equal(t, tc.expected, ready) + }) + } +} + +func TestIsDeploymentReadyErrors(t *testing.T) { + s := &stateSkel{} + + t.Run("marshal error", func(t *testing.T) { + dep := toUnstructuredDeployment(t, &appsv1.Deployment{}) + dep.Object["bad"] = make(chan int) + _, err := s.isDeploymentReady(dep, logr.Discard()) + require.ErrorContains(t, err, "failed to marshall unstructured deployment object") + }) + + t.Run("unmarshal error", func(t *testing.T) { + dep := toUnstructuredDeployment(t, &appsv1.Deployment{}) + dep.Object["status"] = "not-an-object" + _, err := s.isDeploymentReady(dep, logr.Discard()) + require.ErrorContains(t, err, "failed to unmarshall to deployment object") + }) +} + +// --- generic state-object deletion --------------------------------------------- + +// newDeletionSkel builds a stateSkel whose client knows a namespaced kind +// (DaemonSet) and a cluster-scoped kind (ClusterRole); every other supported GVK +// is unmapped and therefore skipped via the NoMatch path. +func newDeletionSkel(t *testing.T, objs ...client.Object) (*stateSkel, client.Client) { + t.Helper() + return newDeletionSkelWithInterceptor(t, interceptor.Funcs{}, objs...) +} + +func newDeletionSkelWithInterceptor(t *testing.T, funcs interceptor.Funcs, objs ...client.Object) (*stateSkel, client.Client) { + t.Helper() + sch := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(sch)) + require.NoError(t, appsv1.AddToScheme(sch)) + require.NoError(t, rbacv1.AddToScheme(sch)) + + mapper := meta.NewDefaultRESTMapper(nil) + mapper.Add(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "DaemonSet"}, meta.RESTScopeNamespace) + mapper.Add(schema.GroupVersionKind{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole"}, meta.RESTScopeRoot) + + cl := fake.NewClientBuilder().WithScheme(sch).WithRESTMapper(mapper). + WithObjects(objs...).WithInterceptorFuncs(funcs).Build() + s := &stateSkel{name: "test-state", namespace: "test-ns", client: cl, scheme: sch} + return s, cl +} + +func labeledDaemonSet(name, ns string) *appsv1.DaemonSet { + return &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: ns, Labels: map[string]string{consts.StateLabel: "test-state"}, + }} +} + +func labeledClusterRole(name string) *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{ + Name: name, Labels: map[string]string{consts.StateLabel: "test-state"}, + }} +} + +func TestDeleteStateRelatedObjectsScoping(t *testing.T) { + s, cl := newDeletionSkel(t, + labeledDaemonSet("ds-here", "test-ns"), + labeledDaemonSet("ds-other", "other-ns"), + labeledClusterRole("cr-1"), + ) + + found, err := s.deleteStateRelatedObjects(context.Background()) + require.NoError(t, err) + assert.True(t, found) + + // Namespaced kinds are listed only in the operator namespace, so the + // same-labeled DaemonSet in another namespace survives. + assert.True(t, apierrors.IsNotFound(cl.Get(context.Background(), + client.ObjectKey{Name: "ds-here", Namespace: "test-ns"}, &appsv1.DaemonSet{}))) + assert.NoError(t, cl.Get(context.Background(), + client.ObjectKey{Name: "ds-other", Namespace: "other-ns"}, &appsv1.DaemonSet{})) + // Cluster-scoped kinds are listed cluster-wide and cleaned up. + assert.True(t, apierrors.IsNotFound(cl.Get(context.Background(), + client.ObjectKey{Name: "cr-1"}, &rbacv1.ClusterRole{}))) +} + +func TestHandleStateObjectsDeletion(t *testing.T) { + t.Run("objects present reports not ready while deleting", func(t *testing.T) { + s, _ := newDeletionSkel(t, labeledDaemonSet("ds-here", "test-ns")) + st, err := s.handleStateObjectsDeletion(context.Background()) + require.NoError(t, err) + assert.Equal(t, SyncState(SyncStateNotReady), st) + }) + t.Run("nothing to delete reports ignore", func(t *testing.T) { + s, _ := newDeletionSkel(t) + st, err := s.handleStateObjectsDeletion(context.Background()) + require.NoError(t, err) + assert.Equal(t, SyncState(SyncStateIgnore), st) + }) + t.Run("deletion error surfaces as sync error", func(t *testing.T) { + s, _ := newDeletionSkelWithInterceptor(t, interceptor.Funcs{ + Delete: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.DeleteOption) error { + return fmt.Errorf("boom delete") + }, + }, labeledDaemonSet("ds-here", "test-ns")) + st, err := s.handleStateObjectsDeletion(context.Background()) + require.ErrorContains(t, err, "failed to delete k8s objects") + assert.Equal(t, SyncState(SyncStateError), st) + }) +} + +// TestDeleteStateRelatedObjectsForbiddenListSkipped characterizes current +// behavior: a Forbidden List is treated as "nothing of this kind to clean up". +// If list permission is revoked after the operand was created (e.g. an RBAC +// change), cleanup silently reports Ignore and leaves the object behind. Update +// this test if Forbidden is promoted to a propagated error. +func TestDeleteStateRelatedObjectsForbiddenListSkipped(t *testing.T) { + s, cl := newDeletionSkelWithInterceptor(t, interceptor.Funcs{ + List: func(_ context.Context, _ client.WithWatch, _ client.ObjectList, _ ...client.ListOption) error { + return apierrors.NewForbidden(schema.GroupResource{Resource: "daemonsets"}, "", fmt.Errorf("nope")) + }, + }, labeledDaemonSet("ds-here", "test-ns")) + + st, err := s.handleStateObjectsDeletion(context.Background()) + require.NoError(t, err) + assert.Equal(t, SyncState(SyncStateIgnore), st) + // The object the operator could not list is left in place. + assert.NoError(t, cl.Get(context.Background(), + client.ObjectKey{Name: "ds-here", Namespace: "test-ns"}, &appsv1.DaemonSet{})) +} + +func TestDeleteStateRelatedObjectsNotFoundOnDeleteIgnored(t *testing.T) { + deleteCalls := 0 + s, _ := newDeletionSkelWithInterceptor(t, interceptor.Funcs{ + Delete: func(_ context.Context, _ client.WithWatch, obj client.Object, _ ...client.DeleteOption) error { + deleteCalls++ + assert.Equal(t, "ds-here", obj.GetName()) + assert.Equal(t, "test-ns", obj.GetNamespace()) + return apierrors.NewNotFound(schema.GroupResource{Resource: "daemonsets"}, obj.GetName()) + }, + }, labeledDaemonSet("ds-here", "test-ns")) + + found, err := s.deleteStateRelatedObjects(context.Background()) + require.NoError(t, err) // NotFound on delete is ignored + assert.True(t, found) + assert.Equal(t, 1, deleteCalls, "expected exactly one delete attempt") +} + +func TestDeleteStateRelatedObjectsListErrorPropagates(t *testing.T) { + s, _ := newDeletionSkelWithInterceptor(t, interceptor.Funcs{ + List: func(_ context.Context, _ client.WithWatch, _ client.ObjectList, _ ...client.ListOption) error { + return fmt.Errorf("boom list") + }, + }, labeledDaemonSet("ds-here", "test-ns")) + + _, err := s.deleteStateRelatedObjects(context.Background()) + require.ErrorContains(t, err, "boom list") +} + +func TestDeleteStateRelatedObjectsDeleteErrorPropagates(t *testing.T) { + s, _ := newDeletionSkelWithInterceptor(t, interceptor.Funcs{ + Delete: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.DeleteOption) error { + return fmt.Errorf("boom delete") + }, + }, labeledDaemonSet("ds-here", "test-ns")) + + _, err := s.deleteStateRelatedObjects(context.Background()) + require.ErrorContains(t, err, "boom delete") +} + +func TestDeleteStateRelatedObjectsSkipsAlreadyDeleting(t *testing.T) { + ds := labeledDaemonSet("ds-deleting", "test-ns") + now := metav1.Now() + ds.DeletionTimestamp = &now + ds.Finalizers = []string{"nvidia.com/keep"} + + deletes := 0 + s, _ := newDeletionSkelWithInterceptor(t, interceptor.Funcs{ + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + deletes++ + return cl.Delete(ctx, obj, opts...) + }, + }, ds) + + found, err := s.deleteStateRelatedObjects(context.Background()) + require.NoError(t, err) + assert.True(t, found, "an object still present counts as found") + assert.Zero(t, deletes, "an object already being deleted must not be deleted again") +} + +// erroringRESTMapper returns a non-NoMatch error from RESTMapping. +// deleteStateRelatedObjects only calls RESTMapping, so the embedded (nil) +// interface is never otherwise exercised. +type erroringRESTMapper struct{ meta.RESTMapper } + +func (erroringRESTMapper) RESTMapping(schema.GroupKind, ...string) (*meta.RESTMapping, error) { + return nil, fmt.Errorf("boom mapping error") +} + +func TestDeleteStateRelatedObjectsMappingErrorPropagates(t *testing.T) { + sch := runtime.NewScheme() + require.NoError(t, appsv1.AddToScheme(sch)) + cl := fake.NewClientBuilder().WithScheme(sch).WithRESTMapper(erroringRESTMapper{}).Build() + s := &stateSkel{name: "test-state", namespace: "test-ns", client: cl, scheme: sch} + + // A RESTMapping error other than NoMatch must propagate rather than be skipped. + _, err := s.deleteStateRelatedObjects(context.Background()) + require.ErrorContains(t, err, "boom mapping error") +} diff --git a/internal/state/state_skel_test.go b/internal/state/state_skel_test.go index 9151530243..7e83de8167 100644 --- a/internal/state/state_skel_test.go +++ b/internal/state/state_skel_test.go @@ -17,16 +17,400 @@ package state import ( + "context" + "fmt" "testing" "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" + "github.com/NVIDIA/gpu-operator/internal/consts" + "github.com/NVIDIA/gpu-operator/internal/utils" ) +func newDeploymentUnstructured(name, ns string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}) + obj.SetName(name) + obj.SetNamespace(ns) + return obj +} + +// skelTestScheme returns a fresh scheme registering exactly the types the +// stateSkel fake clients exercise (ConfigMaps, ServiceAccounts, DaemonSets), +// keeping each test hermetic instead of relying on the global scheme. +func skelTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(s)) + require.NoError(t, appsv1.AddToScheme(s)) + return s +} + +func newTestSkel(t *testing.T, cl client.Client) *stateSkel { + t.Helper() + return &stateSkel{ + name: "test-state", + description: "test description", + namespace: "test-ns", + client: cl, + scheme: skelTestScheme(t), + } +} + +func newConfigMapUnstructured(name, ns string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"}) + obj.SetName(name) + obj.SetNamespace(ns) + _ = unstructured.SetNestedStringMap(obj.Object, map[string]string{"key": "value"}, "data") + return obj +} + +func newServiceAccountUnstructured(name, ns string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ServiceAccount"}) + obj.SetName(name) + obj.SetNamespace(ns) + return obj +} + +func newDaemonSetUnstructured(name, ns string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "DaemonSet"}) + obj.SetName(name) + obj.SetNamespace(ns) + return obj +} + +func setDaemonSetStatus(obj *unstructured.Unstructured, desired, available, updated int64) { + _ = unstructured.SetNestedField(obj.Object, desired, "status", "desiredNumberScheduled") + _ = unstructured.SetNestedField(obj.Object, available, "status", "numberAvailable") + _ = unstructured.SetNestedField(obj.Object, updated, "status", "updatedNumberScheduled") +} + +func TestSkelNameAndDescription(t *testing.T) { + s := newTestSkel(t, fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build()) + assert.Equal(t, "test-state", s.Name()) + assert.Equal(t, "test description", s.Description()) +} + +func TestGetObj(t *testing.T) { + existing := newConfigMapUnstructured("cm-a", "test-ns") + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(existing).Build() + s := newTestSkel(t, cl) + + // Object exists. + got := newConfigMapUnstructured("cm-a", "test-ns") + require.NoError(t, s.getObj(context.Background(), got)) + + // Object does not exist -> IsNotFound error is returned. + missing := newConfigMapUnstructured("cm-missing", "test-ns") + err := s.getObj(context.Background(), missing) + require.True(t, apierrors.IsNotFound(err)) +} + +func TestCreateObj(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build() + s := newTestSkel(t, cl) + + obj := newConfigMapUnstructured("cm-new", "test-ns") + require.NoError(t, s.createObj(context.Background(), obj)) + + // Creating the same object again returns an AlreadyExists error. + err := s.createObj(context.Background(), obj) + require.True(t, apierrors.IsAlreadyExists(err)) +} + +func TestCheckDeleteSupported(t *testing.T) { + s := newTestSkel(t, fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build()) + + // Supported GVK (ConfigMap) - no panic, returns cleanly. + s.checkDeleteSupported(context.Background(), newConfigMapUnstructured("cm", "test-ns")) + + // Unsupported GVK - exercises the warning branch. + unsupported := &unstructured.Unstructured{} + unsupported.SetGroupVersionKind(schema.GroupVersionKind{Group: "custom.io", Version: "v1", Kind: "Widget"}) + unsupported.SetName("w") + s.checkDeleteSupported(context.Background(), unsupported) +} + +func TestUpdateObj(t *testing.T) { + existing := newConfigMapUnstructured("cm-a", "test-ns") + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(existing).Build() + s := newTestSkel(t, cl) + + // Fetch the current object to obtain a valid resourceVersion, then update it. + current := newConfigMapUnstructured("cm-a", "test-ns") + require.NoError(t, s.getObj(context.Background(), current)) + require.NoError(t, unstructured.SetNestedStringMap(current.Object, map[string]string{"key": "updated"}, "data")) + require.NoError(t, s.updateObj(context.Background(), current)) + + // Update error path via interceptor. + errClient := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(existing). + WithInterceptorFuncs(interceptor.Funcs{ + Update: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.UpdateOption) error { + return fmt.Errorf("injected update error") + }, + }).Build() + errSkel := newTestSkel(t, errClient) + err := errSkel.updateObj(context.Background(), newConfigMapUnstructured("cm-a", "test-ns")) + require.ErrorContains(t, err, "failed to update resource") +} + +func TestAddStateSpecificLabels(t *testing.T) { + s := newTestSkel(t, fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build()) + obj := newConfigMapUnstructured("cm", "test-ns") + s.addStateSpecificLabels(obj) + assert.Equal(t, "test-state", obj.GetLabels()[consts.StateLabel]) +} + +func TestMergeObjectsResourceVersion(t *testing.T) { + s := newTestSkel(t, fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build()) + + updated := newConfigMapUnstructured("cm", "test-ns") + current := newConfigMapUnstructured("cm", "test-ns") + current.SetResourceVersion("1234") + + require.NoError(t, s.mergeObjects(updated, current)) + assert.Equal(t, "1234", updated.GetResourceVersion()) +} + +func TestMergeServiceAccount(t *testing.T) { + s := newTestSkel(t, fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build()) + + updated := newServiceAccountUnstructured("sa", "test-ns") + current := newServiceAccountUnstructured("sa", "test-ns") + current.SetResourceVersion("42") + require.NoError(t, unstructured.SetNestedSlice(current.Object, + []interface{}{map[string]interface{}{"name": "sa-token"}}, "secrets")) + require.NoError(t, unstructured.SetNestedSlice(current.Object, + []interface{}{map[string]interface{}{"name": "pull-secret"}}, "imagePullSecrets")) + + require.NoError(t, s.mergeObjects(updated, current)) + + secrets, ok, err := unstructured.NestedSlice(updated.Object, "secrets") + require.NoError(t, err) + require.True(t, ok) + assert.Len(t, secrets, 1) + + pullSecrets, ok, err := unstructured.NestedSlice(updated.Object, "imagePullSecrets") + require.NoError(t, err) + require.True(t, ok) + assert.Len(t, pullSecrets, 1) +} + +func TestCreateOrUpdateObjsCreatesNewObject(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build() + s := newTestSkel(t, cl) + + obj := newDaemonSetUnstructured("ds-a", "test-ns") + noop := func(_ *unstructured.Unstructured) error { return nil } + + require.NoError(t, s.createOrUpdateObjs(context.Background(), noop, []*unstructured.Unstructured{obj})) + + // The DaemonSet should now exist with a hash annotation and state label set. + got := newDaemonSetUnstructured("ds-a", "test-ns") + require.NoError(t, s.getObj(context.Background(), got)) + assert.NotEmpty(t, got.GetAnnotations()[consts.NvidiaAnnotationHashKey]) + assert.Equal(t, "test-state", got.GetLabels()[consts.StateLabel]) +} + +func TestCreateOrUpdateObjsUpdatesExistingObject(t *testing.T) { + existing := newConfigMapUnstructured("cm-a", "test-ns") + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(existing).Build() + s := newTestSkel(t, cl) + + desired := newConfigMapUnstructured("cm-a", "test-ns") + require.NoError(t, unstructured.SetNestedStringMap(desired.Object, map[string]string{"key": "new"}, "data")) + noop := func(_ *unstructured.Unstructured) error { return nil } + + require.NoError(t, s.createOrUpdateObjs(context.Background(), noop, []*unstructured.Unstructured{desired})) + + got := newConfigMapUnstructured("cm-a", "test-ns") + require.NoError(t, s.getObj(context.Background(), got)) + data, found, err := unstructured.NestedStringMap(got.Object, "data") + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, "new", data["key"]) +} + +func TestCreateOrUpdateObjsSkipsUnchangedDaemonSet(t *testing.T) { + // Build the desired object exactly as createOrUpdateObjs would before hashing: + // controller reference is a no-op here, state labels are applied, then the hash + // is computed. Seed a current DaemonSet carrying that same hash so the update + // is skipped. + desired := newDaemonSetUnstructured("ds-a", "test-ns") + s := newTestSkel(t, nil) + s.addStateSpecificLabels(desired) + hash := utils.GetObjectHash(desired) + + current := newDaemonSetUnstructured("ds-a", "test-ns") + current.SetLabels(map[string]string{consts.StateLabel: "test-state"}) + current.SetAnnotations(map[string]string{consts.NvidiaAnnotationHashKey: hash}) + + // Fail the sync if the client is asked to update anything: matching hashes must + // short-circuit before updateObj is ever called. + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(current). + WithInterceptorFuncs(interceptor.Funcs{ + Update: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.UpdateOption) error { + return fmt.Errorf("unexpected update: an unchanged object must not be updated") + }, + }).Build() + s.client = cl + + noop := func(_ *unstructured.Unstructured) error { return nil } + require.NoError(t, s.createOrUpdateObjs(context.Background(), noop, []*unstructured.Unstructured{desired})) +} + +func TestCreateOrUpdateObjsSetControllerReferenceError(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build() + s := newTestSkel(t, cl) + + obj := newConfigMapUnstructured("cm-a", "test-ns") + failRef := func(_ *unstructured.Unstructured) error { return fmt.Errorf("ref error") } + + err := s.createOrUpdateObjs(context.Background(), failRef, []*unstructured.Unstructured{obj}) + require.ErrorContains(t, err, "failed to set controller reference") +} + +func TestCreateOrUpdateObjsCreateError(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(_ context.Context, _ client.WithWatch, _ client.Object, _ ...client.CreateOption) error { + return fmt.Errorf("injected create error") + }, + }).Build() + s := newTestSkel(t, cl) + + obj := newConfigMapUnstructured("cm-a", "test-ns") + noop := func(_ *unstructured.Unstructured) error { return nil } + err := s.createOrUpdateObjs(context.Background(), noop, []*unstructured.Unstructured{obj}) + require.ErrorContains(t, err, "injected create error") +} + +func TestGetSyncState(t *testing.T) { + t.Run("all objects ready", func(t *testing.T) { + cm := newConfigMapUnstructured("cm-a", "test-ns") + daemonSet := newDaemonSetUnstructured("ds-a", "test-ns") + setDaemonSetStatus(daemonSet, 2, 2, 2) + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(cm, daemonSet).Build() + s := newTestSkel(t, cl) + + state, err := s.getSyncState(context.Background(), + []*unstructured.Unstructured{newConfigMapUnstructured("cm-a", "test-ns"), newDaemonSetUnstructured("ds-a", "test-ns")}) + require.NoError(t, err) + assert.Equal(t, SyncState(SyncStateReady), state) + }) + + t.Run("object not found is not ready", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).Build() + s := newTestSkel(t, cl) + state, err := s.getSyncState(context.Background(), + []*unstructured.Unstructured{newConfigMapUnstructured("cm-missing", "test-ns")}) + require.NoError(t, err) + assert.Equal(t, SyncState(SyncStateNotReady), state) + }) + + t.Run("daemonset not ready", func(t *testing.T) { + daemonSet := newDaemonSetUnstructured("ds-a", "test-ns") + setDaemonSetStatus(daemonSet, 3, 1, 1) + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(daemonSet).Build() + s := newTestSkel(t, cl) + state, err := s.getSyncState(context.Background(), + []*unstructured.Unstructured{newDaemonSetUnstructured("ds-a", "test-ns")}) + require.NoError(t, err) + assert.Equal(t, SyncState(SyncStateNotReady), state) + }) + + t.Run("get error propagates", func(t *testing.T) { + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, _ ...client.GetOption) error { + return fmt.Errorf("injected get error") + }, + }).Build() + s := newTestSkel(t, cl) + state, err := s.getSyncState(context.Background(), + []*unstructured.Unstructured{newConfigMapUnstructured("cm-a", "test-ns")}) + require.Error(t, err) + assert.Equal(t, SyncState(SyncStateNotReady), state) + }) + + t.Run("deployment ready", func(t *testing.T) { + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "dep-a", Namespace: "test-ns"}, + Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](1)}, + Status: appsv1.DeploymentStatus{Replicas: 1, UpdatedReplicas: 1, AvailableReplicas: 1}, + } + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(dep).Build() + s := newTestSkel(t, cl) + state, err := s.getSyncState(context.Background(), + []*unstructured.Unstructured{newDeploymentUnstructured("dep-a", "test-ns")}) + require.NoError(t, err) + assert.Equal(t, SyncState(SyncStateReady), state) + }) + + t.Run("deployment not ready", func(t *testing.T) { + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "dep-a", Namespace: "test-ns"}, + Spec: appsv1.DeploymentSpec{Replicas: ptr.To[int32](2)}, + Status: appsv1.DeploymentStatus{Replicas: 1, UpdatedReplicas: 1, AvailableReplicas: 1}, + } + cl := fake.NewClientBuilder().WithScheme(skelTestScheme(t)).WithObjects(dep).Build() + s := newTestSkel(t, cl) + state, err := s.getSyncState(context.Background(), + []*unstructured.Unstructured{newDeploymentUnstructured("dep-a", "test-ns")}) + require.NoError(t, err) + assert.Equal(t, SyncState(SyncStateNotReady), state) + }) +} + +func TestSyncObjects(t *testing.T) { + owner := &nvidiav1alpha1.NVIDIADriver{ObjectMeta: metav1.ObjectMeta{Name: "driver-a", UID: "uid-a"}} + + t.Run("creates objects with owner reference", func(t *testing.T) { + // driverTestScheme registers NVIDIADriver so SetControllerReference resolves the owner. + sch := driverTestScheme(t) + cl := fake.NewClientBuilder().WithScheme(sch).Build() + s := &stateSkel{name: "state-driver", namespace: "test-ns", client: cl, scheme: sch} + + _, err := s.syncObjects(context.Background(), owner, + []*unstructured.Unstructured{newDaemonSetUnstructured("ds-a", "test-ns")}) + require.NoError(t, err) + + dsList := &appsv1.DaemonSetList{} + require.NoError(t, cl.List(context.Background(), dsList)) + require.Len(t, dsList.Items, 1) + require.Len(t, dsList.Items[0].OwnerReferences, 1) + assert.Equal(t, "driver-a", dsList.Items[0].OwnerReferences[0].Name) + }) + + t.Run("set controller reference error", func(t *testing.T) { + // skelTestScheme omits NVIDIADriver, so SetControllerReference on the owner fails. + sch := skelTestScheme(t) + cl := fake.NewClientBuilder().WithScheme(sch).Build() + s := &stateSkel{name: "state-driver", namespace: "test-ns", client: cl, scheme: sch} + + _, err := s.syncObjects(context.Background(), owner, + []*unstructured.Unstructured{newDaemonSetUnstructured("ds-a", "test-ns")}) + require.ErrorContains(t, err, "failed to create/update objects") + }) +} + func toUnstructuredDaemonSet(t *testing.T, ds *appsv1.DaemonSet) *unstructured.Unstructured { t.Helper() obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(ds) @@ -117,3 +501,35 @@ func TestIsDaemonSetReady(t *testing.T) { }) } } + +func TestGetSupportedGVKs(t *testing.T) { + // Generic cleanup deletes every kind in this list, so its exact contents + // matter. Compare the full set (ElementsMatch also flags duplicates and is + // order-independent) rather than probing for a single kind. + expected := []schema.GroupVersionKind{ + {Group: "", Version: "v1", Kind: "ServiceAccount"}, + {Group: "", Version: "v1", Kind: "ConfigMap"}, + {Group: "apps", Version: "v1", Kind: "DaemonSet"}, + {Group: "apps", Version: "v1", Kind: "Deployment"}, + {Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"}, + {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole"}, + {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRoleBinding"}, + {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "Role"}, + {Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "RoleBinding"}, + {Group: "k8s.cni.cncf.io", Version: "v1", Kind: "NetworkAttachmentDefinition"}, + {Group: "batch", Version: "v1", Kind: "CronJob"}, + {Group: "security.openshift.io", Version: "v1", Kind: "SecurityContextConstraints"}, + {Group: "", Version: "v1", Kind: "Pod"}, + {Group: "", Version: "v1", Kind: "Service"}, + {Group: "monitoring.coreos.com", Version: "v1", Kind: "ServiceMonitor"}, + {Group: "scheduling.k8s.io", Version: "v1", Kind: "PriorityClass"}, + {Group: "", Version: "v1", Kind: "Taint"}, + {Group: "policy", Version: "v1beta1", Kind: "PodSecurityPolicy"}, + {Group: "node.k8s.io", Version: "v1", Kind: "RuntimeClass"}, + {Group: "monitoring.coreos.com", Version: "v1", Kind: "PrometheusRule"}, + {Group: "resource.k8s.io", Version: "v1", Kind: "ResourceClaimTemplate"}, + {Group: "resource.k8s.io", Version: "v1beta2", Kind: "ResourceClaimTemplate"}, + {Group: "resource.k8s.io", Version: "v1beta1", Kind: "ResourceClaimTemplate"}, + } + assert.ElementsMatch(t, expected, getSupportedGVKs()) +}