diff --git a/cmd/nvidia-validator/main.go b/cmd/nvidia-validator/main.go index c5d1ca7cbc..a144ede505 100644 --- a/cmd/nvidia-validator/main.go +++ b/cmd/nvidia-validator/main.go @@ -18,6 +18,7 @@ package main import ( "context" + "errors" "fmt" "os" "os/exec" @@ -48,6 +49,7 @@ import ( nvidiav1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1" nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1" + "github.com/NVIDIA/gpu-operator/internal/consts" "github.com/NVIDIA/gpu-operator/internal/driver" "github.com/NVIDIA/gpu-operator/internal/info" "github.com/NVIDIA/gpu-operator/internal/utils" @@ -234,6 +236,10 @@ const ( gpuWorkloadConfigVMVgpu = "vm-vgpu" // CCCapableLabelKey represents NFD label name to indicate if the node is capable to run CC workloads CCCapableLabelKey = "nvidia.com/cc.capable" + // devicePluginDeployLabelKey gates whether the nvidia-device-plugin DaemonSet + // is scheduled on the node. Users set it to "false" to disable the plugin per + // node (for example, to hand GPU allocation off to a DRA driver). + devicePluginDeployLabelKey = "nvidia.com/gpu.deploy.device-plugin" // appComponentLabelKey indicates the label key of the component appComponentLabelKey = "app.kubernetes.io/component" // wslNvidiaSMIPath indicates the path to the nvidia-smi binary on WSL @@ -1193,6 +1199,10 @@ func (p *Plugin) validate() error { p.setKubeClient(kubeClient) err = p.validateGPUResource() + if errors.Is(err, errDevicePluginDisabled) { + log.Info("Device plugin is disabled on this node, skipping GPU resource validation") + return nil + } if err != nil { return err } @@ -1444,7 +1454,43 @@ func (p *Plugin) countGPUResources() (int64, error) { return count, nil } +// errDevicePluginDisabled signals validate() that the device plugin is not +// expected to publish GPU capacity on this node, so plugin validation should be +// skipped without failing the operator-validator pod. Returned by +// validateGPUResource when isDevicePluginDisabledOnNode holds for the node. +var errDevicePluginDisabled = errors.New("device plugin is disabled on this node") + +// isDevicePluginDisabledOnNode reports whether the nvidia-device-plugin is +// intentionally not running on the given node, either because a user explicitly +// disabled it via nvidia.com/gpu.deploy.device-plugin=false or because the node +// has been switched to the DRA resource-allocation stack. In either case the +// classic operator-validator's plugin-validation step must not wait for GPU +// capacity that will never appear. +func isDevicePluginDisabledOnNode(node *corev1.Node) bool { + if node == nil { + return false + } + nodeLabels := node.GetLabels() + if nodeLabels[devicePluginDeployLabelKey] == "false" { + return true + } + if nodeLabels[consts.GPUAllocationModeLabelKey] == string(consts.GPUAllocationModeDRA) { + return true + } + return false +} + func (p *Plugin) validateGPUResource() error { + // Skip the retry loop entirely when the device plugin is not expected to + // advertise GPU capacity on this node (per-node disable or DRA mode). + node, err := getNode(p.ctx, p.kubeClient) + if err != nil { + return fmt.Errorf("unable to fetch node by name %s to check for GPU resources: %s", nodeNameFlag, err) + } + if isDevicePluginDisabledOnNode(node) { + return errDevicePluginDisabled + } + for retry := 1; retry <= gpuResourceDiscoveryWaitRetries; retry++ { // get node info to check discovered GPU resources node, err := getNode(p.ctx, p.kubeClient) diff --git a/cmd/nvidia-validator/main_test.go b/cmd/nvidia-validator/main_test.go index 8885a9dc30..a9410eb850 100644 --- a/cmd/nvidia-validator/main_test.go +++ b/cmd/nvidia-validator/main_test.go @@ -26,6 +26,8 @@ import ( "testing" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestResolveHostNvidiaSMI(t *testing.T) { @@ -376,3 +378,84 @@ UNKNOWN_FEATURE: true`, }) } } + +func TestIsDevicePluginDisabledOnNode(t *testing.T) { + tests := []struct { + name string + node *corev1.Node + wantOK bool + }{ + { + name: "nil node is treated as plugin-enabled", + node: nil, + wantOK: false, + }, + { + name: "node with no labels is plugin-enabled", + node: &corev1.Node{}, + wantOK: false, + }, + { + name: "device-plugin label absent, other labels present", + node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{ + "foo": "bar", + }}}, + wantOK: false, + }, + { + name: "device-plugin explicitly true keeps validation active", + node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{ + "nvidia.com/gpu.deploy.device-plugin": "true", + }}}, + wantOK: false, + }, + { + name: "device-plugin explicitly false triggers skip", + node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{ + "nvidia.com/gpu.deploy.device-plugin": "false", + }}}, + wantOK: true, + }, + { + name: "device-plugin empty value is not treated as disabled", + node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{ + "nvidia.com/gpu.deploy.device-plugin": "", + }}}, + wantOK: false, + }, + { + name: "device-plugin paused value is not treated as disabled", + node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{ + "nvidia.com/gpu.deploy.device-plugin": "paused-for-driver-upgrade", + }}}, + wantOK: false, + }, + { + name: "resource-allocation mode dra triggers skip", + node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{ + "nvidia.com/gpu-operator.resource-allocation.mode": "dra", + }}}, + wantOK: true, + }, + { + name: "resource-allocation mode device-plugin does not skip", + node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{ + "nvidia.com/gpu-operator.resource-allocation.mode": "device-plugin", + }}}, + wantOK: false, + }, + { + name: "both signals set to skip still skips", + node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{ + "nvidia.com/gpu.deploy.device-plugin": "false", + "nvidia.com/gpu-operator.resource-allocation.mode": "dra", + }}}, + wantOK: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.wantOK, isDevicePluginDisabledOnNode(tt.node)) + }) + } +}