Fix DRA kind e2e setup and queue resource accounting

Signed-off-by: xuwentao <cutenear1993@yahoo.com>

Author: xu-wentao

Makefile

@@ -224,7 +224,7 @@ e2e-test-cronjob: images
 	E2E_TYPE=CRONJOB ./hack/run-e2e-kind.sh
 
 e2e-test-dra: images
-	E2E_TYPE=DRA FEATURE_GATES="DynamicResourceAllocation=true" ./hack/run-e2e-kind.sh
+	E2E_TYPE=DRA FEATURE_GATES="DynamicResourceAllocation=true,DRAConsumableCapacity=true" ./hack/run-e2e-kind.sh
 
 e2e-test-hypernode: images
 	E2E_TYPE=HYPERNODE ./hack/run-e2e-kind.sh

docs/design/capacity-dra-support.md

@@ -151,6 +151,8 @@ In practice:
 
 These two responsibilities are complementary rather than overlapping.
 
+For consumable capacity, Volcano accounts the logical total. A request for two devices with `8Gi` of memory each consumes `16Gi` of the queue's `memory.deviceclass/<DeviceClass>` quota. This does not solve physical fragmentation: if the cluster has two `8Gi` devices, a single Pod that needs `10Gi` on one device can still pass queue quota while failing the final DRA allocation.
+
 ## Root Queue and Total Resource View
 
 For DRA resources, the cluster-wide total can come from actual cluster state or from an administrator’s explicit queue configuration.

docs/user-guide/how_to_use_dra_quota.md

@@ -53,9 +53,14 @@ data:
       - name: drf
       - name: predicates
       - name: capacity
+        arguments:
+          capacity.DynamicResourceAllocationEnable: true
+          capacity.DRAConsumableCapacityEnable: true
       - name: nodeorder
 ```
 
+The capacity plugin arguments are optional. When they are not set, Volcano uses the corresponding Kubernetes feature gate values as defaults.
+
 ## Queue Key Formats
 
 Use the following keys in `spec.capability`, `spec.deserved`, and `spec.guarantee`.
@@ -67,6 +72,8 @@ Use the following keys in `spec.capability`, `spec.deserved`, and `spec.guarante
 
 These keys can coexist with normal resources such as `cpu`, `memory`, and `nvidia.com/gpu`.
 
+Consumable-capacity requests are counted per allocated device. For example, a ResourceClaim requesting `count: 2` and `memory: 8Gi` consumes `16Gi` of the queue key `memory.deviceclass/<DeviceClass>`.
+
 ## Queue Configuration Examples
 
 ### Whole-Card GPU Queue
@@ -303,6 +310,8 @@ The following request styles are not the main quota-accounting target:
 
 If your environment relies heavily on those modes, evaluate behavior carefully before treating them as strict queue quota signals.
 
+Queue quota is a logical total-resource check. It does not guarantee that the requested capacity can be placed on a specific physical device layout. For example, a queue may have `16Gi` of total GPU memory quota, while one Pod requesting `10Gi` still cannot fit onto any single `8Gi` device. Kubernetes DRA and the installed driver make the final physical allocation decision.
+
 ## Best Practices
 
 1. Use `ResourceClaimTemplate` for per-Pod allocation lifecycle

hack/e2e-kind-config.yaml

@@ -18,14 +18,49 @@ nodes:
         apiVersion: kubelet.config.k8s.io/v1beta1
         kind: KubeletConfiguration
         containerLogMaxSize: "50Mi"
+        featureGates:
+          DynamicResourceAllocation: true
+          DRAConsumableCapacity: true
       - |
         kind: ClusterConfiguration
         apiServer:
           extraArgs:
             runtime-config: "admissionregistration.k8s.io/v1beta1"
             enable-admission-plugins: "MutatingAdmissionPolicy"
+            feature-gates: "MutatingAdmissionPolicy=true,DynamicResourceAllocation=true,DRAConsumableCapacity=true"
+        controllerManager:
+          extraArgs:
+            feature-gates: "MutatingAdmissionPolicy=true,DynamicResourceAllocation=true,DRAConsumableCapacity=true"
   # the four workers
   - role: worker
+    kubeadmConfigPatches:
+      - |
+        apiVersion: kubelet.config.k8s.io/v1beta1
+        kind: KubeletConfiguration
+        featureGates:
+          DynamicResourceAllocation: true
+          DRAConsumableCapacity: true
+  - role: worker
+    kubeadmConfigPatches:
+      - |
+        apiVersion: kubelet.config.k8s.io/v1beta1
+        kind: KubeletConfiguration
+        featureGates:
+          DynamicResourceAllocation: true
+          DRAConsumableCapacity: true
   - role: worker
+    kubeadmConfigPatches:
+      - |
+        apiVersion: kubelet.config.k8s.io/v1beta1
+        kind: KubeletConfiguration
+        featureGates:
+          DynamicResourceAllocation: true
+          DRAConsumableCapacity: true
   - role: worker
-  - role: worker
+    kubeadmConfigPatches:
+      - |
+        apiVersion: kubelet.config.k8s.io/v1beta1
+        kind: KubeletConfiguration
+        featureGates:
+          DynamicResourceAllocation: true
+          DRAConsumableCapacity: true

hack/lib/install.sh

@@ -19,7 +19,7 @@ function kind-up-cluster {
   check-kind
 
   echo "Running kind: [kind create cluster ${CLUSTER_CONTEXT[*]} ${KIND_OPT}]"
-  kind create cluster "${CLUSTER_CONTEXT[@]}" ${KIND_OPT}
+  kind create cluster "${CLUSTER_CONTEXT[@]}" ${KIND_OPT} || exit 1
 
   echo
   check-images
@@ -33,6 +33,32 @@ function kind-up-cluster {
   if [[ "${E2E_TYPE}" == "AGENTSCHEDULER" ]]; then
     kind load docker-image ${IMAGE_PREFIX}/vc-agent-scheduler:${TAG} "${CLUSTER_CONTEXT[@]}" --nodes ${CLUSTER_CONTEXT[1]}-control-plane
   fi
+  if [[ "${E2E_TYPE}" == "DRA" || "${E2E_TYPE}" == "ALL" ]]; then
+    ensure-dra-test-images
+  fi
+}
+
+function ensure-dra-test-images {
+  local dra_images=(
+    "nginx:1.29.3-alpine"
+    "registry.k8s.io/kwok/kwok:v0.7.0"
+    "registry.k8s.io/sig-storage/hostpathplugin:v1.16.1"
+  )
+
+  echo
+  echo "Ensuring DRA test images are available locally"
+  for image in "${dra_images[@]}"; do
+    if ! docker image inspect "${image}" >/dev/null 2>&1; then
+      echo "Pulling image ${image} ..."
+      docker pull "${image}" >/dev/null || exit 1
+    fi
+  done
+
+  echo
+  echo "Loading DRA test images into kind cluster"
+  for image in "${dra_images[@]}"; do
+    kind load docker-image "${image}" "${CLUSTER_CONTEXT[@]}" || exit 1
+  done
 }
 
 # check if the required images exist
@@ -81,6 +107,17 @@ function check-kind {
   if [[ $? -ne 0 ]]; then
     echo "Installing kind ..."
     GOOS=${OS} go install sigs.k8s.io/kind@v0.31.0
+    local bin_path
+    bin_path=$(go env GOBIN)
+    if [[ -z "${bin_path}" ]]; then
+      bin_path="$(go env GOPATH)/bin"
+    fi
+    export PATH="${bin_path}:${PATH}"
+    if ! command -v kind >/dev/null 2>&1; then
+      echo -e "\033[31mERROR\033[0m: kind installation completed but the binary is still not available on PATH"
+      exit 1
+    fi
+    echo -n "Using kind, version: " && kind version
   else
     echo -n "Found kind, version: " && kind version
   fi

hack/run-e2e-kind.sh

@@ -24,6 +24,7 @@ export LOG_LEVEL=3
 export CLEANUP_CLUSTER=${CLEANUP_CLUSTER:-1}
 export E2E_TYPE=${E2E_TYPE:-"ALL"}
 export ARTIFACTS_PATH=${ARTIFACTS_PATH:-"${VK_ROOT}/volcano-e2e-logs"}
+DRA_GINKGO_FOCUS=${DRA_GINKGO_FOCUS:-"DRA (Quota )?E2E Test"}
 mkdir -p "$ARTIFACTS_PATH"
 
 NAMESPACE=${NAMESPACE:-volcano-system}
@@ -379,7 +380,7 @@ case ${E2E_TYPE} in
     KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --skip="\[sig-.*\]" --slow-spec-threshold='30s' --progress ./test/e2e/schedulingaction/
     KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/vcctl/
     KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/cronjob/
-    KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress --focus="DRA E2E Test" ./test/e2e/dra/
+    KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress --focus="${DRA_GINKGO_FOCUS}" ./test/e2e/dra/
     KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/admission/
     KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -r --slow-spec-threshold='30s' --progress ./test/e2e/hypernode/
     ;;
@@ -414,7 +415,7 @@ case ${E2E_TYPE} in
     ;;
 "DRA")
     echo "Running dra e2e suite..."
-    KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --slow-spec-threshold='30s' --progress --focus="DRA E2E Test" ./test/e2e/dra/
+    KUBECONFIG=${KUBECONFIG} GOOS=${OS} ginkgo -v -r --slow-spec-threshold='30s' --progress --focus="${DRA_GINKGO_FOCUS}" ./test/e2e/dra/
     ;;
 "ADMISSION_POLICY")
     echo "Running admission policy e2e suite..."

pkg/scheduler/api/job_info.go

@@ -1446,40 +1446,19 @@ func (ji *JobInfo) GetMinDRAResources() map[string]*DRAResource {
 	}
 
 	result := make(map[string]*DRAResource)
-	processedRoles := make(map[string]struct{})
-
-	// Since DRA requests can vary per task/pod, we aggregate them based on TaskMinAvailable
-	for _, task := range ji.Tasks {
-		if task.DRAResreq == nil {
-			continue
-		}
-
-		// Calculate how many times this task type needs to run
-		taskType := task.TaskRole
-		minNum, ok := ji.TaskMinAvailable[taskType]
-		if !ok || minNum <= 0 {
-			continue
-		}
-		if _, seen := processedRoles[taskType]; seen {
-			continue
-		}
-		processedRoles[taskType] = struct{}{}
-
-		for deviceClass, res := range task.DRAResreq {
+	addResource := func(res map[string]*DRAResource, times int32) {
+		for deviceClass, request := range res {
 			if _, exists := result[deviceClass]; !exists {
 				result[deviceClass] = &DRAResource{
 					Count:    0,
 					Capacity: make(map[string]resource.Quantity),
 				}
 			}
 
-			result[deviceClass].Count += res.Count * int64(minNum)
-			for dim, cap := range res.Capacity {
+			result[deviceClass].Count += request.Count * int64(times)
+			for dim, cap := range request.Capacity {
 				totalCap := cap.DeepCopy()
-				// resource.Quantity has no Multiply func, so we parse memory/cpu as MilliValues
-				// For exact values we can just use set
-				// Since Quantity can represent fractional, we will loop to add
-				for i := int32(0); i < minNum-1; i++ {
+				for i := int32(0); i < times-1; i++ {
 					totalCap.Add(cap)
 				}
 
@@ -1493,6 +1472,62 @@ func (ji *JobInfo) GetMinDRAResources() map[string]*DRAResource {
 		}
 	}
 
+	if len(ji.TaskMinAvailable) == 0 {
+		minAvailable := ji.MinAvailable
+		if minAvailable <= 0 {
+			return nil
+		}
+
+		tasks := make([]*TaskInfo, 0, len(ji.Tasks))
+		for _, task := range ji.Tasks {
+			if task.DRAResreq != nil {
+				tasks = append(tasks, task)
+			}
+		}
+		sort.Slice(tasks, func(i, j int) bool {
+			if tasks[i].Namespace != tasks[j].Namespace {
+				return tasks[i].Namespace < tasks[j].Namespace
+			}
+			if tasks[i].Name != tasks[j].Name {
+				return tasks[i].Name < tasks[j].Name
+			}
+			return tasks[i].UID < tasks[j].UID
+		})
+
+		for i, task := range tasks {
+			if int32(i) >= minAvailable {
+				break
+			}
+			addResource(task.DRAResreq, 1)
+		}
+
+		if len(result) == 0 {
+			return nil
+		}
+		return result
+	}
+
+	// Since DRA requests can vary per task/pod, we aggregate them based on TaskMinAvailable
+	processedRoles := make(map[string]struct{})
+	for _, task := range ji.Tasks {
+		if task.DRAResreq == nil {
+			continue
+		}
+
+		// Calculate how many times this task type needs to run
+		taskType := task.TaskRole
+		minNum, ok := ji.TaskMinAvailable[taskType]
+		if !ok || minNum <= 0 {
+			continue
+		}
+		if _, seen := processedRoles[taskType]; seen {
+			continue
+		}
+		processedRoles[taskType] = struct{}{}
+
+		addResource(task.DRAResreq, minNum)
+	}
+
 	if len(result) == 0 {
 		return nil
 	}

pkg/scheduler/api/job_info_test.go

@@ -718,3 +718,70 @@ func TestParseMinMemberInfoChanged(t *testing.T) {
 		})
 	}
 }
+
+func TestGetMinDRAResourcesIdempotent(t *testing.T) {
+	job := NewJobInfo("test-job")
+	job.TaskMinAvailable["worker"] = 2
+	job.Tasks[TaskID("worker-0")] = &TaskInfo{
+		UID:      TaskID("worker-0"),
+		Name:     "worker-0",
+		TaskRole: "worker",
+		DRAResreq: map[string]*DRAResource{
+			"gpu.com": {
+				Count: 1,
+				Capacity: map[string]resource.Quantity{
+					"memory": resource.MustParse("8Gi"),
+				},
+			},
+		},
+	}
+
+	expected := map[string]*DRAResource{
+		"gpu.com": {
+			Count: 2,
+			Capacity: map[string]resource.Quantity{
+				"memory": resource.MustParse("16Gi"),
+			},
+		},
+	}
+
+	assert.Equal(t, expected, job.GetMinDRAResources())
+	assert.Equal(t, expected, job.GetMinDRAResources())
+	assert.Equal(t, map[string]int32{"worker": 2}, job.TaskMinAvailable)
+}
+
+func TestGetMinDRAResourcesFallbackWithoutTaskMinAvailable(t *testing.T) {
+	job := NewJobInfo("test-job")
+	job.MinAvailable = 2
+	job.Tasks[TaskID("task-b")] = &TaskInfo{
+		UID:       TaskID("task-b"),
+		Namespace: "ns",
+		Name:      "task-b",
+		DRAResreq: map[string]*DRAResource{
+			"gpu.com": {Count: 1},
+		},
+	}
+	job.Tasks[TaskID("task-a")] = &TaskInfo{
+		UID:       TaskID("task-a"),
+		Namespace: "ns",
+		Name:      "task-a",
+		DRAResreq: map[string]*DRAResource{
+			"gpu.com": {Count: 2},
+		},
+	}
+	job.Tasks[TaskID("task-c")] = &TaskInfo{
+		UID:       TaskID("task-c"),
+		Namespace: "ns",
+		Name:      "task-c",
+		DRAResreq: map[string]*DRAResource{
+			"gpu.com": {Count: 4},
+		},
+	}
+
+	assert.Equal(t, map[string]*DRAResource{
+		"gpu.com": {
+			Count:    3,
+			Capacity: map[string]resource.Quantity{},
+		},
+	}, job.GetMinDRAResources())
+}