reduce move_pages granularity

pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go

@@ -120,7 +120,7 @@ func (p *DynamicPolicy) sharedCoresWithoutNUMABindingAllocationHandler(_ context
 			RequestQuantity:                  reqFloat64,
 		}
 
-		if !shouldRampUp {
+		if !shouldRampUp || allocationInfo.GetSpecifiedPoolName() == "bmq" {
 			targetPoolName := allocationInfo.GetSpecifiedPoolName()
 			poolAllocationInfo := p.state.GetAllocationInfo(targetPoolName, state.FakedContainerName)
 
@@ -723,7 +723,7 @@ func (p *DynamicPolicy) adjustPoolsAndIsolatedEntries(poolsQuantityMap map[strin
 		return fmt.Errorf("reclaimOverlapShareRatio failed with error: %v", err)
 	}
 
-	poolsCPUSet, isolatedCPUSet, err := p.generatePoolsAndIsolation(poolsQuantityMap, isolatedQuantityMap, availableCPUs, reclaimOverlapShareRatio)
+	poolsCPUSet, isolatedCPUSet, err := p.generatePoolsAndIsolation(poolsQuantityMap, entries, isolatedQuantityMap, availableCPUs, reclaimOverlapShareRatio)
 	if err != nil {
 		return fmt.Errorf("generatePoolsAndIsolation failed with error: %v", err)
 	}
@@ -1084,6 +1084,7 @@ func (p *DynamicPolicy) generateNUMABindingPoolsCPUSetInPlace(poolsCPUSet map[st
 // 2. use the left cores to allocate among different pools
 // 3. apportion to other pools if reclaimed is disabled
 func (p *DynamicPolicy) generatePoolsAndIsolation(poolsQuantityMap map[string]map[int]int,
+	entries state.PodEntries,
 	isolatedQuantityMap map[string]map[string]int, availableCPUs machine.CPUSet,
 	reclaimOverlapShareRatio map[string]float64) (poolsCPUSet map[string]machine.CPUSet,
 	isolatedCPUSet map[string]map[string]machine.CPUSet, err error,
@@ -1204,6 +1205,30 @@ func (p *DynamicPolicy) generatePoolsAndIsolation(poolsQuantityMap map[string]ma
 	} else if nonBindingPoolsTotalQuantity > 0 {
 		general.Infof("can't allocate for all pools")
 
+		var podEntriesQuantityMap map[string]map[int]int
+		podEntriesQuantityMap, tErr = state.GetSharedQuantityMapFromPodEntries(entries, nil)
+		if tErr != nil {
+			err = fmt.Errorf("get pools quantity from pod entries with error: %v", tErr)
+			return
+		}
+
+		poolsRequestQuantityMap := make(map[string]int)
+		for poolName, numaToQuantity := range podEntriesQuantityMap {
+			if len(numaToQuantity) == 1 {
+				for numaID, quantity := range numaToQuantity {
+					if numaID == state.FakedNUMAID && quantity != 0 {
+						poolsRequestQuantityMap[poolName] = quantity
+					}
+				}
+			}
+		}
+
+		// revise pool quantity for un-expandable pool
+		requestQuantity, requestOk := poolsRequestQuantityMap["bmq"]
+		if requestOk && nonBindingAvailableSize > 0 {
+			nonBindingPoolsQuantityMap["bmq"] = int(math.Ceil((float64(requestQuantity) / float64(nonBindingAvailableSize)) * float64(nonBindingPoolsTotalQuantity)))
+		}
+
 		nonBindingAvailableCPUs, tErr = p.generateProportionalPoolsCPUSetInPlace(nonBindingPoolsQuantityMap, poolsCPUSet, nonBindingAvailableCPUs)
 
 		if tErr != nil {
@@ -1376,7 +1401,7 @@ func (p *DynamicPolicy) apportionReclaimedPool(poolsCPUSet map[string]machine.CP
 	}
 
 	for poolName, poolCPUs := range poolsCPUSet {
-		if state.ResidentPools.Has(poolName) {
+		if state.ResidentPools.Has(poolName) || poolName == "bmq" {
 			continue
 		} else if _, found := nonBindingPoolsQuantityMap[poolName]; !found {
 			// numa-binding && none-reclaimed pools already handled in generateNUMABindingPoolsCPUSetInPlace

pkg/agent/qrm-plugins/memory/dynamicpolicy/util_linux_amd64.go

@@ -50,11 +50,11 @@ const (
 	// GetNumaForPagesMaxEachTime get numa for 16384 pages(64MB) at most at a time
 	GetNumaForPagesMaxEachTime = 16384
 
-	// MovePagesMaxEachTime means move 5120 pages(20MB) at most at a time
-	MovePagesMaxEachTime = 5120
+	// MovePagesMaxEachTime means move 1280 pages(5MB) at most at a time
+	MovePagesMaxEachTime = 1280
 
-	// MovePagesMinEachTime means move 256 pages(1MB) at least at a time
-	MovePagesMinEachTime = 256
+	// MovePagesMinEachTime means move 64 pages(256 KB) at least at a time
+	MovePagesMinEachTime = 64
 
 	// MovePagesAcceptableTimeCost is acceptable time cost of each move pages is 20ms
 	MovePagesAcceptableTimeCost = 20
@@ -265,12 +265,19 @@ func MovePagesForProcess(ctx context.Context, procDir string, pid int, srcNumas
 	pagesMargin := GetNumaForPagesMaxEachTime
 	var pagesAdrr []uint64
 	var phyPagesAddr []uint64
+	var maxGetProcessPagesNuma int64
 
 	getPhyPagesOnSourceNumas := func() {
+		t0 := time.Now()
 		pagesNuma, err := getProcessPagesNuma(int32(pid), uint64(len(pagesAdrr)), pagesAdrr)
 		if err != nil {
 			return
 		}
+
+		if time.Since(t0).Milliseconds() > maxGetProcessPagesNuma {
+			maxGetProcessPagesNuma = time.Since(t0).Milliseconds()
+		}
+
 		for i, n := range pagesNuma {
 			if srcNumasBitSet.IsSet(int(n)) {
 				pageAddr := pagesAdrr[i]
@@ -291,6 +298,8 @@ func MovePagesForProcess(ctx context.Context, procDir string, pid int, srcNumas
 		}
 	}
 
+	general.Infof("getProcessPagesNuma pid: %d, timeCost max: %d ms\n", pid, maxGetProcessPagesNuma)
+
 	// handle left pagesAddr whose length less than GetNumaForPagesMaxEachTime
 	if len(pagesAdrr) > 0 {
 		getPhyPagesOnSourceNumas()
@@ -359,7 +368,9 @@ func moveProcessPagesToOneNuma(ctx context.Context, pid int32, pagesAddr []uint6
 	leftPhyPages := pagesAddr[:]
 
 	var movePagesLatencyMax int64
+	var movePagesLenWhenLatencyMax int
 	movePagesEachTime := MovePagesMinEachTime
+	moveCount := 0
 	var movingPagesAddr []uint64
 
 	var errList []error
@@ -390,6 +401,7 @@ pagesLoop:
 			errList = append(errList, fmt.Errorf("failed move pages for pid: %d, numa: %d err: %v", pid, dstNuma, err))
 			continue
 		}
+		moveCount++
 		timeCost := time.Since(start).Milliseconds()
 
 		if timeCost == 0 {
@@ -399,6 +411,7 @@ pagesLoop:
 
 		if timeCost > movePagesLatencyMax {
 			movePagesLatencyMax = timeCost
+			movePagesLenWhenLatencyMax = len(movingPagesAddr)
 		}
 
 		movePagesEachTime = movePagesEachTime * MovePagesAcceptableTimeCost / int(timeCost)
@@ -410,7 +423,7 @@ pagesLoop:
 	}
 
 	if movePagesLatencyMax > 0 {
-		general.Infof("moveProcessPagesToOneNuma pid: %d, dest numa: %d, timeCost max: %d ms\n", pid, dstNuma, movePagesLatencyMax)
+		general.Infof("moveProcessPagesToOneNuma pid: %d, dest numa: %d, moveCount: %d, timeCost max: %d ms, movePages len: %d\n", pid, dstNuma, moveCount, movePagesLatencyMax, movePagesLenWhenLatencyMax)
 	}
 	return utilerrors.NewAggregate(errList)
 }

pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/provisionassembler/assembler_common.go

@@ -241,6 +241,11 @@ func (pa *ProvisionAssemblerCommon) AssembleProvision() (types.InternalCPUCalcul
 		isolationPoolSizes = isolationLowerSizes
 	}
 
+	if size, ok := sharePoolRequirements["bmq"]; ok {
+		isolationPoolSizes["bmq"] = size
+		delete(sharePoolRequirements, "bmq")
+	}
+
 	shareAndIsolatePoolSizes, poolThrottled := regulatePoolSizes(sharePoolRequirements, isolationPoolSizes, shareAndIsolatedPoolAvailable, nodeEnableReclaim, *pa.allowSharedCoresOverlapReclaimedCores)
 	for _, r := range *pa.regionMap {
 		if r.Type() == configapi.QoSRegionTypeShare && !r.IsNumaBinding() {

pkg/agent/sysadvisor/types/consts.go

@@ -37,8 +37,8 @@ const (
 	MinDedicatedCPURequirement = 4
 
 	MaxRampUpStep     = 10
-	MaxRampDownStep   = 2
-	MinRampDownPeriod = 30 * time.Second
+	MaxRampDownStep   = 100
+	MinRampDownPeriod = 0 * time.Second
 	StartUpPeriod     = 30 * time.Second
 
 	ReclaimUsageMarginForOverlap = 6