webhook: support elasticquota enable update resource key

Signed-off-by: lijunxin <[email protected]>

pkg/features/features.go

@@ -68,6 +68,11 @@ const (
 	// to belong to the users and will not be preempted back.
 	ElasticQuotaGuaranteeUsage featuregate.Feature = "ElasticQuotaGuaranteeUsage"
 
+	// ElasticQuotaEnableUpdateResourceKey allows to update resource key in standard operation
+	// when delete resource type: from child to parent
+	// when add resource type: from parent to child
+	ElasticQuotaEnableUpdateResourceKey featuregate.Feature = "ElasticQuotaEnableUpdateResourceKey"
+
 	// DisableDefaultQuota disable default quota.
 	DisableDefaultQuota featuregate.Feature = "DisableDefaultQuota"
 
@@ -94,6 +99,7 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
 	MultiQuotaTree:                         {Default: false, PreRelease: featuregate.Alpha},
 	ElasticQuotaIgnorePodOverhead:          {Default: false, PreRelease: featuregate.Alpha},
 	ElasticQuotaGuaranteeUsage:             {Default: false, PreRelease: featuregate.Alpha},
+	ElasticQuotaEnableUpdateResourceKey:    {Default: false, PreRelease: featuregate.Alpha},
 	DisableDefaultQuota:                    {Default: false, PreRelease: featuregate.Alpha},
 	SupportParentQuotaSubmitPod:            {Default: false, PreRelease: featuregate.Alpha},
 	EnableQuotaAdmission:                   {Default: false, PreRelease: featuregate.Alpha},

pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go

@@ -563,24 +563,33 @@ func (gqm *GroupQuotaManager) updateOneGroupSharedWeightNoLock(quotaInfo *QuotaI
 	gqm.runtimeQuotaCalculatorMap[quotaInfo.ParentName].updateOneGroupSharedWeight(quotaInfo)
 }
 
+// updateResourceKeyNoLock based on quotaInfo.CalculateInfo.Max of self
+// Note: RootQuotaName need to be updated as allResourceKeys
 func (gqm *GroupQuotaManager) updateResourceKeyNoLock() {
 	// collect all dimensions
-	resourceKeys := make(map[v1.ResourceName]struct{})
+	allResourceKeys := make(map[v1.ResourceName]struct{})
 	for quotaName, quotaInfo := range gqm.quotaInfoMap {
-		if quotaName == extension.DefaultQuotaName || quotaName == extension.SystemQuotaName {
+		if quotaName == extension.RootQuotaName || quotaName == extension.DefaultQuotaName || quotaName == extension.SystemQuotaName {
 			continue
 		}
+		resourceKeys := make(map[v1.ResourceName]struct{})
 		for resName := range quotaInfo.CalculateInfo.Max {
+			allResourceKeys[resName] = struct{}{}
 			resourceKeys[resName] = struct{}{}
 		}
-	}
-
-	if !reflect.DeepEqual(resourceKeys, gqm.resourceKeys) {
-		gqm.resourceKeys = resourceKeys
-		for _, runtimeQuotaCalculator := range gqm.runtimeQuotaCalculatorMap {
+		// update right now
+		if runtimeQuotaCalculator, ok := gqm.runtimeQuotaCalculatorMap[quotaName]; ok && runtimeQuotaCalculator != nil && !reflect.DeepEqual(resourceKeys, runtimeQuotaCalculator.resourceKeys) {
 			runtimeQuotaCalculator.updateResourceKeys(resourceKeys)
 		}
 	}
+
+	if !reflect.DeepEqual(allResourceKeys, gqm.resourceKeys) {
+		gqm.resourceKeys = allResourceKeys
+	}
+	// in case RootQuota-resourceKey is nil
+	if runtimeQuotaCalculator, ok := gqm.runtimeQuotaCalculatorMap[extension.RootQuotaName]; ok && runtimeQuotaCalculator != nil && !reflect.DeepEqual(allResourceKeys, runtimeQuotaCalculator.resourceKeys) {
+		runtimeQuotaCalculator.updateResourceKeys(allResourceKeys)
+	}
 }
 
 func (gqm *GroupQuotaManager) GetAllQuotaNames() map[string]struct{} {
@@ -1019,7 +1028,6 @@ func (gqm *GroupQuotaManager) updateQuotaInternalNoLock(newQuotaInfo, oldQuotaIn
 
 	// update resource keys
 	gqm.updateResourceKeyNoLock()
-
 	oldMin := v1.ResourceList{}
 	if oldQuotaInfo != nil {
 		oldMin = oldQuotaInfo.CalculateInfo.Min

pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go

@@ -40,7 +40,8 @@ import (
 )
 
 const (
-	GigaByte = 1024 * 1048576
+	GigaByte                = 1024 * 1048576
+	ExtendedResourceKeyXCPU = "x-cpu"
 )
 
 func TestGroupQuotaManager_QuotaAdd(t *testing.T) {
@@ -151,6 +152,7 @@ func TestGroupQuotaManager_UpdateQuota(t *testing.T) {
 }
 
 func TestGroupQuotaManager_UpdateQuotaInternalAndRequest(t *testing.T) {
+	// add resource to node
 	gqm := NewGroupQuotaManagerForTest()
 	deltaRes := createResourceList(96, 160*GigaByte)
 	gqm.UpdateClusterTotalResource(deltaRes)
@@ -160,23 +162,61 @@ func TestGroupQuotaManager_UpdateQuotaInternalAndRequest(t *testing.T) {
 
 	AddQuotaToManager(t, gqm, "test1", extension.RootQuotaName, 96, 160*GigaByte, 50, 80*GigaByte, true, false)
 
-	// test1 request[120, 290]  runtime == maxQuota
+	// request[120, 290] > maxQuota, runtime == maxQuota
 	request := createResourceList(120, 290*GigaByte)
 	gqm.updateGroupDeltaRequestNoLock("test1", request, request, 0)
 	runtime := gqm.RefreshRuntime("test1")
-	assert.Equal(t, deltaRes, runtime)
+	expectCurrentRuntime := deltaRes
+	assert.Equal(t, expectCurrentRuntime, runtime)
 
+	// update resourceKey
 	quota1 := CreateQuota("test1", extension.RootQuotaName, 64, 100*GigaByte, 60, 90*GigaByte, true, false)
 	quota1.Labels[extension.LabelQuotaIsParent] = "false"
 	err := gqm.UpdateQuota(quota1, false)
 	assert.Nil(t, err)
 	quotaInfo := gqm.GetQuotaInfoByName("test1")
 	assert.Equal(t, createResourceList(64, 100*GigaByte), quotaInfo.CalculateInfo.Max)
+	runtime = gqm.RefreshRuntime("test1")
+	expectCurrentRuntime = createResourceList(64, 100*GigaByte)
+	assert.Equal(t, expectCurrentRuntime, runtime)
 
+	// added max ExtendedResourceKeyXCPU without node resource added
+	// runtime.ExtendedResourceKeyXCPU = 0
+	request[ExtendedResourceKeyXCPU] = *resource.NewQuantity(80, resource.DecimalSI)
+	gqm.updateGroupDeltaRequestNoLock("test1", request, request, 0)
+	xCPUQuantity := resource.NewQuantity(100, resource.DecimalSI)
+	quota1.Spec.Max[ExtendedResourceKeyXCPU] = *xCPUQuantity
+	maxJson, err := json.Marshal(quota1.Spec.Max)
+	assert.Nil(t, err)
+	quota1.Annotations[extension.AnnotationSharedWeight] = string(maxJson)
+	gqm.UpdateQuota(quota1, false)
+	quotaInfo = gqm.quotaInfoMap["test1"]
+	assert.True(t, quotaInfo != nil)
+	assert.Equal(t, *xCPUQuantity, quotaInfo.CalculateInfo.Max[ExtendedResourceKeyXCPU])
 	runtime = gqm.RefreshRuntime("test1")
-	assert.Equal(t, createResourceList(64, 100*GigaByte), runtime)
-}
+	expectCurrentRuntime[ExtendedResourceKeyXCPU] = resource.Quantity{Format: resource.DecimalSI}
+	assert.Equal(t, expectCurrentRuntime, runtime)
 
+	// add ExtendedResourceKeyXCPU to node resource
+	deltaRes[ExtendedResourceKeyXCPU] = *xCPUQuantity
+	gqm.UpdateClusterTotalResource(deltaRes)
+	runtime = gqm.RefreshRuntime("test1")
+	expectCurrentRuntime[ExtendedResourceKeyXCPU] = *resource.NewQuantity(80, resource.DecimalSI)
+	assert.Equal(t, expectCurrentRuntime, runtime)
+
+	// delete max ExtendedResourceKeyXCPU
+	delete(quota1.Spec.Max, ExtendedResourceKeyXCPU)
+	maxJson, err = json.Marshal(quota1.Spec.Max)
+	assert.Nil(t, err)
+	quota1.Annotations[extension.AnnotationSharedWeight] = string(maxJson)
+	gqm.UpdateQuota(quota1, false)
+	quotaInfo = gqm.quotaInfoMap["test1"]
+	assert.True(t, quotaInfo != nil)
+	assert.Equal(t, resource.Quantity{}, quotaInfo.CalculateInfo.Max[ExtendedResourceKeyXCPU])
+	runtime = gqm.RefreshRuntime("test1")
+	delete(expectCurrentRuntime, ExtendedResourceKeyXCPU)
+	assert.Equal(t, expectCurrentRuntime, runtime)
+}
 func TestGroupQuotaManager_DeleteOneGroup(t *testing.T) {
 	gqm := NewGroupQuotaManagerForTest()
 	gqm.UpdateClusterTotalResource(createResourceList(1000, 1000*GigaByte))

pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator.go

@@ -21,6 +21,7 @@ import (
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apiserver/pkg/util/flowcontrol/format"
 	"k8s.io/klog/v2"
 
 	"github.com/koordinator-sh/koordinator/pkg/util"
@@ -127,6 +128,7 @@ func (qt *quotaTree) redistribution(totalResource int64) {
 		if node.request > min {
 			// if a node's request > autoScaleMin, the node needs adjustQuota
 			// the node's runtime is autoScaleMin
+			klog.Infof("[jxtest] node.request: %v,min: %v,node.sharedWeight: %v", node.request, min, node.sharedWeight)
 			needAdjustQuotaNodes = append(needAdjustQuotaNodes, node)
 			totalSharedWeight += node.sharedWeight
 			node.runtimeQuota = min
@@ -139,6 +141,7 @@ func (qt *quotaTree) redistribution(totalResource int64) {
 				node.runtimeQuota = min
 			}
 		}
+		klog.Infof("[jxtest] toPartitionResource: %v", toPartitionResource)
 		toPartitionResource -= node.runtimeQuota
 	}
 
@@ -152,7 +155,7 @@ func (qt *quotaTree) iterationForRedistribution(totalRes, totalSharedWeight int6
 		// if totalSharedWeight is not larger than 0, no need to iterate anymore.
 		return
 	}
-
+	klog.Infof("[jxtest] iterationForRedistribution")
 	needAdjustQuotaNodes := make([]*quotaNode, 0)
 	toPartitionResource, needAdjustTotalSharedWeight := int64(0), int64(0)
 	for _, node := range nodes {
@@ -493,6 +496,7 @@ func (qtw *RuntimeQuotaCalculator) calculateRuntimeNoLock() {
 	//lock outside
 	for resKey := range qtw.resourceKeys {
 		totalResourcePerKey := *qtw.totalResource.Name(resKey, resource.DecimalSI)
+		klog.Infof("[jxtest] key: %v,q:%v", resKey, format.ToJSON(totalResourcePerKey))
 		qtw.quotaTree[resKey].redistribution(getQuantityValue(totalResourcePerKey, resKey))
 	}
 }

pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator_test.go

@@ -31,6 +31,13 @@ import (
 	"github.com/koordinator-sh/koordinator/apis/thirdparty/scheduler-plugins/pkg/apis/scheduling/v1alpha1"
 )
 
+const (
+	TestNode1 = "node1"
+	TestNode2 = "node2"
+	TestNode3 = "node3"
+	TestNode4 = "node4"
+)
+
 func TestQuotaInfo_GetLimitRequest(t *testing.T) {
 	max := createResourceList(100, 10000)
 	req := createResourceList(1000, 1000)
@@ -129,29 +136,134 @@ func createElasticQuota() *v1alpha1.ElasticQuota {
 	return eQ
 }
 
-func TestRuntimeQuotaCalculator_Iteration4AdjustQuota(t *testing.T) {
-	qtw := NewRuntimeQuotaCalculator("testTreeName")
-	resourceKey := make(map[corev1.ResourceName]struct{})
-	cpu := corev1.ResourceCPU
-	resourceKey[cpu] = struct{}{}
-	qtw.updateResourceKeys(resourceKey)
-	qtw.quotaTree[cpu].insert("node1", 40, 5, 10, 0, true)
-	qtw.quotaTree[cpu].insert("node2", 60, 20, 15, 0, true)
-	qtw.quotaTree[cpu].insert("node3", 50, 40, 20, 0, true)
-	qtw.quotaTree[cpu].insert("node4", 80, 70, 15, 0, true)
-	qtw.totalResource = corev1.ResourceList{}
-	qtw.totalResource[corev1.ResourceCPU] = *resource.NewMilliQuantity(100, resource.DecimalSI)
-	qtw.calculateRuntimeNoLock()
-	if qtw.globalRuntimeVersion == 0 {
-		t.Error("error")
+func TestRuntimeQuotaCalculator_IterationAdjustQuota(t *testing.T) {
+	type quotaNodeInfo = struct {
+		groupName         string
+		sharedWeight      int64
+		request           int64
+		min               int64
+		guarantee         int64
+		allowLentResource bool
 	}
-	if qtw.quotaTree[cpu].quotaNodes["node1"].runtimeQuota != 5 ||
-		qtw.quotaTree[cpu].quotaNodes["node2"].runtimeQuota != 20 ||
-		qtw.quotaTree[cpu].quotaNodes["node3"].runtimeQuota != 35 ||
-		qtw.quotaTree[cpu].quotaNodes["node4"].runtimeQuota != 40 {
-		t.Error("error")
+	node1 := &quotaNodeInfo{
+		groupName:         TestNode1,
+		sharedWeight:      40,
+		request:           5,
+		min:               10,
+		guarantee:         0,
+		allowLentResource: true,
+	}
+	node2 := &quotaNodeInfo{
+		groupName:         TestNode2,
+		sharedWeight:      60,
+		request:           20,
+		min:               15,
+		guarantee:         0,
+		allowLentResource: true,
+	}
+	node3 := &quotaNodeInfo{
+		groupName:         TestNode3,
+		sharedWeight:      50,
+		request:           40,
+		min:               20,
+		guarantee:         0,
+		allowLentResource: true,
+	}
+	node4 := &quotaNodeInfo{
+		groupName:         TestNode4,
+		sharedWeight:      80,
+		request:           70,
+		min:               15,
+		guarantee:         0,
+		allowLentResource: true,
+	}
+	node4_1 := &quotaNodeInfo{
+		groupName:         TestNode4,
+		sharedWeight:      0,
+		request:           70,
+		min:               15,
+		guarantee:         0,
+		allowLentResource: true,
+	}
+	node4_2 := &quotaNodeInfo{
+		groupName:         TestNode4,
+		sharedWeight:      0,
+		request:           70,
+		min:               15,
+		guarantee:         45,
+		allowLentResource: true,
 	}
 
+	testCases := []struct {
+		name              string
+		totalResource     corev1.ResourceList
+		nodes             []*quotaNodeInfo
+		expectedRuntimeMp map[string]map[corev1.ResourceName]int64
+	}{
+		{
+			name: "case1-no-guarantee",
+			totalResource: corev1.ResourceList{
+				corev1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI),
+			},
+			nodes: []*quotaNodeInfo{node1, node2, node3, node4},
+			expectedRuntimeMp: map[string]map[corev1.ResourceName]int64{
+				TestNode1: {corev1.ResourceCPU: 5},
+				TestNode2: {corev1.ResourceCPU: 20},
+				TestNode3: {corev1.ResourceCPU: 35},
+				TestNode4: {corev1.ResourceCPU: 40},
+			},
+		},
+		{
+			name: "case2-node4.sharedWeight=0",
+			totalResource: corev1.ResourceList{
+				corev1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI),
+			},
+			nodes: []*quotaNodeInfo{node1, node2, node3, node4_1},
+			expectedRuntimeMp: map[string]map[corev1.ResourceName]int64{
+				TestNode1: {corev1.ResourceCPU: 5},
+				TestNode2: {corev1.ResourceCPU: 20},
+				TestNode3: {corev1.ResourceCPU: 40},
+				TestNode4: {corev1.ResourceCPU: 15},
+			},
+		},
+		{
+			name: "case3-node4.guarantee>min",
+			totalResource: corev1.ResourceList{
+				corev1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI),
+			},
+			nodes: []*quotaNodeInfo{node1, node2, node3, node4_2},
+			expectedRuntimeMp: map[string]map[corev1.ResourceName]int64{
+				TestNode1: {corev1.ResourceCPU: 5},
+				TestNode2: {corev1.ResourceCPU: 20},
+				TestNode3: {corev1.ResourceCPU: 30},
+				TestNode4: {corev1.ResourceCPU: 45},
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			qtw := NewRuntimeQuotaCalculator("testTreeName")
+			resourceKey := make(map[corev1.ResourceName]struct{})
+			for key, value := range tc.totalResource {
+				if !value.IsZero() {
+					resourceKey[key] = struct{}{}
+				}
+			}
+			qtw.updateResourceKeys(resourceKey)
+			qtw.totalResource = tc.totalResource
+			for _, node := range tc.nodes {
+				for resKey := range resourceKey {
+					qtw.quotaTree[resKey].insert(node.groupName, node.sharedWeight, node.request, node.min, node.guarantee, node.allowLentResource)
+				}
+			}
+			qtw.calculateRuntimeNoLock()
+			for node, rq := range tc.expectedRuntimeMp {
+				for resKey, q := range rq {
+					assert.Equal(t, q, qtw.quotaTree[resKey].quotaNodes[node].runtimeQuota)
+				}
+			}
+		})
+	}
 }
 
 func createQuotaInfoWithRes(name string, max, min corev1.ResourceList) *QuotaInfo {