diff --git a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go index b2c6d30a3..bb40d0e32 100644 --- a/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go +++ b/pkg/agent/qrm-plugins/cpu/dynamicpolicy/policy_allocation_handlers.go @@ -120,7 +120,7 @@ func (p *DynamicPolicy) sharedCoresWithoutNUMABindingAllocationHandler(_ context RequestQuantity: reqFloat64, } - if !shouldRampUp { + if !shouldRampUp || allocationInfo.GetSpecifiedPoolName() == "bmq" { targetPoolName := allocationInfo.GetSpecifiedPoolName() poolAllocationInfo := p.state.GetAllocationInfo(targetPoolName, state.FakedContainerName) @@ -723,7 +723,7 @@ func (p *DynamicPolicy) adjustPoolsAndIsolatedEntries(poolsQuantityMap map[strin return fmt.Errorf("reclaimOverlapShareRatio failed with error: %v", err) } - poolsCPUSet, isolatedCPUSet, err := p.generatePoolsAndIsolation(poolsQuantityMap, isolatedQuantityMap, availableCPUs, reclaimOverlapShareRatio) + poolsCPUSet, isolatedCPUSet, err := p.generatePoolsAndIsolation(poolsQuantityMap, entries, isolatedQuantityMap, availableCPUs, reclaimOverlapShareRatio) if err != nil { return fmt.Errorf("generatePoolsAndIsolation failed with error: %v", err) } @@ -1084,6 +1084,7 @@ func (p *DynamicPolicy) generateNUMABindingPoolsCPUSetInPlace(poolsCPUSet map[st // 2. use the left cores to allocate among different pools // 3. apportion to other pools if reclaimed is disabled func (p *DynamicPolicy) generatePoolsAndIsolation(poolsQuantityMap map[string]map[int]int, + entries state.PodEntries, isolatedQuantityMap map[string]map[string]int, availableCPUs machine.CPUSet, reclaimOverlapShareRatio map[string]float64) (poolsCPUSet map[string]machine.CPUSet, isolatedCPUSet map[string]map[string]machine.CPUSet, err error, @@ -1204,6 +1205,30 @@ func (p *DynamicPolicy) generatePoolsAndIsolation(poolsQuantityMap map[string]ma } else if nonBindingPoolsTotalQuantity > 0 { general.Infof("can't allocate for all pools") + var podEntriesQuantityMap map[string]map[int]int + podEntriesQuantityMap, tErr = state.GetSharedQuantityMapFromPodEntries(entries, nil) + if tErr != nil { + err = fmt.Errorf("get pools quantity from pod entries with error: %v", tErr) + return + } + + poolsRequestQuantityMap := make(map[string]int) + for poolName, numaToQuantity := range podEntriesQuantityMap { + if len(numaToQuantity) == 1 { + for numaID, quantity := range numaToQuantity { + if numaID == state.FakedNUMAID && quantity != 0 { + poolsRequestQuantityMap[poolName] = quantity + } + } + } + } + + // revise pool quantity for un-expandable pool + requestQuantity, requestOk := poolsRequestQuantityMap["bmq"] + if requestOk && nonBindingAvailableSize > 0 { + nonBindingPoolsQuantityMap["bmq"] = int(math.Ceil((float64(requestQuantity) / float64(nonBindingAvailableSize)) * float64(nonBindingPoolsTotalQuantity))) + } + nonBindingAvailableCPUs, tErr = p.generateProportionalPoolsCPUSetInPlace(nonBindingPoolsQuantityMap, poolsCPUSet, nonBindingAvailableCPUs) if tErr != nil { @@ -1376,7 +1401,7 @@ func (p *DynamicPolicy) apportionReclaimedPool(poolsCPUSet map[string]machine.CP } for poolName, poolCPUs := range poolsCPUSet { - if state.ResidentPools.Has(poolName) { + if state.ResidentPools.Has(poolName) || poolName == "bmq" { continue } else if _, found := nonBindingPoolsQuantityMap[poolName]; !found { // numa-binding && none-reclaimed pools already handled in generateNUMABindingPoolsCPUSetInPlace diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/util_linux_amd64.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/util_linux_amd64.go index 58d45f7d5..0364839df 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/util_linux_amd64.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/util_linux_amd64.go @@ -50,11 +50,11 @@ const ( // GetNumaForPagesMaxEachTime get numa for 16384 pages(64MB) at most at a time GetNumaForPagesMaxEachTime = 16384 - // MovePagesMaxEachTime means move 5120 pages(20MB) at most at a time - MovePagesMaxEachTime = 5120 + // MovePagesMaxEachTime means move 1280 pages(5MB) at most at a time + MovePagesMaxEachTime = 1280 - // MovePagesMinEachTime means move 256 pages(1MB) at least at a time - MovePagesMinEachTime = 256 + // MovePagesMinEachTime means move 64 pages(256 KB) at least at a time + MovePagesMinEachTime = 64 // MovePagesAcceptableTimeCost is acceptable time cost of each move pages is 20ms MovePagesAcceptableTimeCost = 20 @@ -265,12 +265,19 @@ func MovePagesForProcess(ctx context.Context, procDir string, pid int, srcNumas pagesMargin := GetNumaForPagesMaxEachTime var pagesAdrr []uint64 var phyPagesAddr []uint64 + var maxGetProcessPagesNuma int64 getPhyPagesOnSourceNumas := func() { + t0 := time.Now() pagesNuma, err := getProcessPagesNuma(int32(pid), uint64(len(pagesAdrr)), pagesAdrr) if err != nil { return } + + if time.Since(t0).Milliseconds() > maxGetProcessPagesNuma { + maxGetProcessPagesNuma = time.Since(t0).Milliseconds() + } + for i, n := range pagesNuma { if srcNumasBitSet.IsSet(int(n)) { pageAddr := pagesAdrr[i] @@ -291,6 +298,8 @@ func MovePagesForProcess(ctx context.Context, procDir string, pid int, srcNumas } } + general.Infof("getProcessPagesNuma pid: %d, timeCost max: %d ms\n", pid, maxGetProcessPagesNuma) + // handle left pagesAddr whose length less than GetNumaForPagesMaxEachTime if len(pagesAdrr) > 0 { getPhyPagesOnSourceNumas() @@ -359,7 +368,9 @@ func moveProcessPagesToOneNuma(ctx context.Context, pid int32, pagesAddr []uint6 leftPhyPages := pagesAddr[:] var movePagesLatencyMax int64 + var movePagesLenWhenLatencyMax int movePagesEachTime := MovePagesMinEachTime + moveCount := 0 var movingPagesAddr []uint64 var errList []error @@ -390,6 +401,7 @@ pagesLoop: errList = append(errList, fmt.Errorf("failed move pages for pid: %d, numa: %d err: %v", pid, dstNuma, err)) continue } + moveCount++ timeCost := time.Since(start).Milliseconds() if timeCost == 0 { @@ -399,6 +411,7 @@ pagesLoop: if timeCost > movePagesLatencyMax { movePagesLatencyMax = timeCost + movePagesLenWhenLatencyMax = len(movingPagesAddr) } movePagesEachTime = movePagesEachTime * MovePagesAcceptableTimeCost / int(timeCost) @@ -410,7 +423,7 @@ pagesLoop: } if movePagesLatencyMax > 0 { - general.Infof("moveProcessPagesToOneNuma pid: %d, dest numa: %d, timeCost max: %d ms\n", pid, dstNuma, movePagesLatencyMax) + general.Infof("moveProcessPagesToOneNuma pid: %d, dest numa: %d, moveCount: %d, timeCost max: %d ms, movePages len: %d\n", pid, dstNuma, moveCount, movePagesLatencyMax, movePagesLenWhenLatencyMax) } return utilerrors.NewAggregate(errList) } diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/provisionassembler/assembler_common.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/provisionassembler/assembler_common.go index 631fc0718..4c78321e3 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/provisionassembler/assembler_common.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/assembler/provisionassembler/assembler_common.go @@ -241,6 +241,11 @@ func (pa *ProvisionAssemblerCommon) AssembleProvision() (types.InternalCPUCalcul isolationPoolSizes = isolationLowerSizes } + if size, ok := sharePoolRequirements["bmq"]; ok { + isolationPoolSizes["bmq"] = size + delete(sharePoolRequirements, "bmq") + } + shareAndIsolatePoolSizes, poolThrottled := regulatePoolSizes(sharePoolRequirements, isolationPoolSizes, shareAndIsolatedPoolAvailable, nodeEnableReclaim, *pa.allowSharedCoresOverlapReclaimedCores) for _, r := range *pa.regionMap { if r.Type() == configapi.QoSRegionTypeShare && !r.IsNumaBinding() { diff --git a/pkg/agent/sysadvisor/types/consts.go b/pkg/agent/sysadvisor/types/consts.go index ac979fc92..8a6a48699 100644 --- a/pkg/agent/sysadvisor/types/consts.go +++ b/pkg/agent/sysadvisor/types/consts.go @@ -37,8 +37,8 @@ const ( MinDedicatedCPURequirement = 4 MaxRampUpStep = 10 - MaxRampDownStep = 2 - MinRampDownPeriod = 30 * time.Second + MaxRampDownStep = 100 + MinRampDownPeriod = 0 * time.Second StartUpPeriod = 30 * time.Second ReclaimUsageMarginForOverlap = 6