From 6dfa76565db613178688b99489e65c9b8125d05c Mon Sep 17 00:00:00 2001 From: Zachary Zhao Date: Sun, 25 Oct 2026 03:07:24 +0800 Subject: [PATCH] fix(api): accept the binder-translated NvFractions limit next to gpu-fraction In NvFractions mode the binder turns gpu-memory.portion.limit into the nvidia.com/container..gpu-memory.limit annotation at bind time and keeps gpu-fraction on the pod. ValidateGPUFractionRequest then rejected that update as "cannot combine ... limit annotation with gpu-fraction", so every bind of a pod with a portion limit failed and the pod cycled between Scheduled and BindingError. Exempt the translated limit when the portion-limit annotation is present, mirroring the existing gpu-memory.limit shorthand exemption. Ref kai-scheduler/KAI-Scheduler#2490 Co-Authored-By: Claude Fable 5.1 Signed-off-by: Zachary Zhao --- .../unreleased/fixed-20261025-025826.yaml | 7 +++ utilities/resources/gpu_sharing_validation.go | 6 +- .../resources/gpu_sharing_validation_test.go | 61 +++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 .changes/unreleased/fixed-20261025-025826.yaml diff --git a/.changes/unreleased/fixed-20261025-025826.yaml b/.changes/unreleased/fixed-20261025-025826.yaml new file mode 100644 index 0000000..1d49007 --- /dev/null +++ b/.changes/unreleased/fixed-20261025-025826.yaml @@ -0,0 +1,7 @@ +kind: Fixed +body: |- + Allow the binder-translated NvFractions limit next to gpu-fraction when gpu-memory.portion.limit is set + +custom: + Author: ZeXi0308 + Issue: "28" diff --git a/utilities/resources/gpu_sharing_validation.go b/utilities/resources/gpu_sharing_validation.go index 2ae5fe2..d933424 100644 --- a/utilities/resources/gpu_sharing_validation.go +++ b/utilities/resources/gpu_sharing_validation.go @@ -56,7 +56,11 @@ func ValidateGPUFractionRequest(pod *v1.Pod) error { // A legacy gpu-memory request may accompany a translated shorthand limit. _, hasMemoryLimitShorthand := pod.Annotations[constants.GpuMemoryLimit] - if req.Limit != nil && (hasLegacyFraction || (hasLegacyMemory && !hasMemoryLimitShorthand)) { + // A legacy gpu-fraction request may accompany the NvFractions limit the binder + // derives from gpu-memory.portion.limit at bind time. + _, _, hasPortionLimit := ExtractGpuMemoryPortionLimitAnnotation(pod) + if req.Limit != nil && + ((hasLegacyFraction && !hasPortionLimit) || (hasLegacyMemory && !hasMemoryLimitShorthand)) { return fmt.Errorf("cannot combine %s limit annotation with %s or %s annotation", constants.NvFractionsAnnotationPrefix, constants.GpuFraction, constants.GpuMemory) } diff --git a/utilities/resources/gpu_sharing_validation_test.go b/utilities/resources/gpu_sharing_validation_test.go index 6d4c59a..9cf9ff9 100644 --- a/utilities/resources/gpu_sharing_validation_test.go +++ b/utilities/resources/gpu_sharing_validation_test.go @@ -177,6 +177,67 @@ func TestValidateGpuMemoryPortionLimitAnnotation(t *testing.T) { } } +func TestValidateGPUFractionRequest_TranslatedLimits(t *testing.T) { + nvRequestKey := CalcGpuFractionAnnotationForContainer("container-0") + nvLimitKey := CalcGpuFractionLimitAnnotationForContainer("container-0") + portionLimitKey := CalcGpuMemoryPortionLimitAnnotationForContainer("container-0") + + tests := []struct { + name string + annotations map[string]string + wantErrContaining string + }{ + { + name: "gpu-fraction with portion limit accepts the binder-translated NvFractions limit", + annotations: map[string]string{ + constants.GpuFraction: "0.5", + portionLimitKey: "0.8", + nvRequestKey: "1000Mi", + nvLimitKey: "1600Mi", + }, + }, + { + name: "gpu-fraction without portion limit rejects an NvFractions limit", + annotations: map[string]string{ + constants.GpuFraction: "0.5", + nvRequestKey: "1000Mi", + nvLimitKey: "1600Mi", + }, + wantErrContaining: "cannot combine", + }, + { + name: "gpu-memory with gpu-memory.limit accepts the translated NvFractions limit", + annotations: map[string]string{ + constants.GpuMemory: "1000", + constants.GpuMemoryLimit: "1600Mi", + nvRequestKey: "1000Mi", + nvLimitKey: "1600Mi", + }, + }, + { + name: "gpu-memory without gpu-memory.limit rejects an NvFractions limit", + annotations: map[string]string{ + constants.GpuMemory: "1000", + nvRequestKey: "1000Mi", + nvLimitKey: "1600Mi", + }, + wantErrContaining: "cannot combine", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Annotations: tt.annotations}, + Spec: v1.PodSpec{Containers: []v1.Container{{Name: "container-0"}}}, + } + + err := ValidateGPUFractionRequest(pod) + assertErrorContains(t, err, tt.wantErrContaining) + }) + } +} + func TestValidateGPUFractionRequest_ComputeSharingMode(t *testing.T) { tests := []struct { name string