Skip GPU resource validation when device plugin is disabled on the node

cmd/nvidia-validator/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -48,6 +49,7 @@ import (
 
 	nvidiav1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1"
 	nvidiav1alpha1 "github.com/NVIDIA/gpu-operator/api/nvidia/v1alpha1"
+	"github.com/NVIDIA/gpu-operator/internal/consts"
 	"github.com/NVIDIA/gpu-operator/internal/driver"
 	"github.com/NVIDIA/gpu-operator/internal/info"
 	"github.com/NVIDIA/gpu-operator/internal/utils"
@@ -234,6 +236,10 @@ const (
 	gpuWorkloadConfigVMVgpu        = "vm-vgpu"
 	// CCCapableLabelKey represents NFD label name to indicate if the node is capable to run CC workloads
 	CCCapableLabelKey = "nvidia.com/cc.capable"
+	// devicePluginDeployLabelKey gates whether the nvidia-device-plugin DaemonSet
+	// is scheduled on the node. Users set it to "false" to disable the plugin per
+	// node (for example, to hand GPU allocation off to a DRA driver).
+	devicePluginDeployLabelKey = "nvidia.com/gpu.deploy.device-plugin"
 	// appComponentLabelKey indicates the label key of the component
 	appComponentLabelKey = "app.kubernetes.io/component"
 	// wslNvidiaSMIPath indicates the path to the nvidia-smi binary on WSL
@@ -1193,6 +1199,10 @@ func (p *Plugin) validate() error {
 	p.setKubeClient(kubeClient)
 
 	err = p.validateGPUResource()
+	if errors.Is(err, errDevicePluginDisabled) {
+		log.Info("Device plugin is disabled on this node, skipping GPU resource validation")
+		return nil
+	}
 	if err != nil {
 		return err
 	}
@@ -1444,7 +1454,43 @@ func (p *Plugin) countGPUResources() (int64, error) {
 	return count, nil
 }
 
+// errDevicePluginDisabled signals validate() that the device plugin is not
+// expected to publish GPU capacity on this node, so plugin validation should be
+// skipped without failing the operator-validator pod. Returned by
+// validateGPUResource when isDevicePluginDisabledOnNode holds for the node.
+var errDevicePluginDisabled = errors.New("device plugin is disabled on this node")
+
+// isDevicePluginDisabledOnNode reports whether the nvidia-device-plugin is
+// intentionally not running on the given node, either because a user explicitly
+// disabled it via nvidia.com/gpu.deploy.device-plugin=false or because the node
+// has been switched to the DRA resource-allocation stack. In either case the
+// classic operator-validator's plugin-validation step must not wait for GPU
+// capacity that will never appear.
+func isDevicePluginDisabledOnNode(node *corev1.Node) bool {
+	if node == nil {
+		return false
+	}
+	nodeLabels := node.GetLabels()
+	if nodeLabels[devicePluginDeployLabelKey] == "false" {
+		return true
+	}
+	if nodeLabels[consts.GPUAllocationModeLabelKey] == string(consts.GPUAllocationModeDRA) {
+		return true
+	}
+	return false
+}
+
 func (p *Plugin) validateGPUResource() error {
+	// Skip the retry loop entirely when the device plugin is not expected to
+	// advertise GPU capacity on this node (per-node disable or DRA mode).
+	node, err := getNode(p.ctx, p.kubeClient)
+	if err != nil {
+		return fmt.Errorf("unable to fetch node by name %s to check for GPU resources: %s", nodeNameFlag, err)
+	}
+	if isDevicePluginDisabledOnNode(node) {
+		return errDevicePluginDisabled
+	}
+
 	for retry := 1; retry <= gpuResourceDiscoveryWaitRetries; retry++ {
 		// get node info to check discovered GPU resources
 		node, err := getNode(p.ctx, p.kubeClient)

cmd/nvidia-validator/main_test.go

@@ -26,6 +26,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestResolveHostNvidiaSMI(t *testing.T) {
@@ -376,3 +378,84 @@ UNKNOWN_FEATURE: true`,
 		})
 	}
 }
+
+func TestIsDevicePluginDisabledOnNode(t *testing.T) {
+	tests := []struct {
+		name   string
+		node   *corev1.Node
+		wantOK bool
+	}{
+		{
+			name:   "nil node is treated as plugin-enabled",
+			node:   nil,
+			wantOK: false,
+		},
+		{
+			name:   "node with no labels is plugin-enabled",
+			node:   &corev1.Node{},
+			wantOK: false,
+		},
+		{
+			name: "device-plugin label absent, other labels present",
+			node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{
+				"foo": "bar",
+			}}},
+			wantOK: false,
+		},
+		{
+			name: "device-plugin explicitly true keeps validation active",
+			node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{
+				"nvidia.com/gpu.deploy.device-plugin": "true",
+			}}},
+			wantOK: false,
+		},
+		{
+			name: "device-plugin explicitly false triggers skip",
+			node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{
+				"nvidia.com/gpu.deploy.device-plugin": "false",
+			}}},
+			wantOK: true,
+		},
+		{
+			name: "device-plugin empty value is not treated as disabled",
+			node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{
+				"nvidia.com/gpu.deploy.device-plugin": "",
+			}}},
+			wantOK: false,
+		},
+		{
+			name: "device-plugin paused value is not treated as disabled",
+			node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{
+				"nvidia.com/gpu.deploy.device-plugin": "paused-for-driver-upgrade",
+			}}},
+			wantOK: false,
+		},
+		{
+			name: "resource-allocation mode dra triggers skip",
+			node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{
+				"nvidia.com/gpu-operator.resource-allocation.mode": "dra",
+			}}},
+			wantOK: true,
+		},
+		{
+			name: "resource-allocation mode device-plugin does not skip",
+			node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{
+				"nvidia.com/gpu-operator.resource-allocation.mode": "device-plugin",
+			}}},
+			wantOK: false,
+		},
+		{
+			name: "both signals set to skip still skips",
+			node: &corev1.Node{ObjectMeta: meta_v1.ObjectMeta{Labels: map[string]string{
+				"nvidia.com/gpu.deploy.device-plugin":              "false",
+				"nvidia.com/gpu-operator.resource-allocation.mode": "dra",
+			}}},
+			wantOK: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.wantOK, isDevicePluginDisabledOnNode(tt.node))
+		})
+	}
+}