From cf5695e98471683d5226b6731ec8a98a5ff3ce16 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 17 Aug 2023 17:46:02 +0100 Subject: [PATCH] HPCC-30118 Fix numWorkersPerPod, workers should not split resourced memory Signed-off-by: Jake Smith --- helm/hpcc/values.yaml | 16 +++++++++------- thorlcr/master/thmastermain.cpp | 23 ++++++++++------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/helm/hpcc/values.yaml b/helm/hpcc/values.yaml index 829511b211d..b4d1b7c9b81 100644 --- a/helm/hpcc/values.yaml +++ b/helm/hpcc/values.yaml @@ -753,13 +753,15 @@ roxie: #ldapUser: roxie_file_access #add system username for accessing files #egress: engineEgress -## The [manager/worker/eclAgent]Resources define the resource limits for each pod. -## workerMemory defines the memory requirements for each individual worker. -## If numWorkersPerPod is >1 (must be a factor of numWorkers) then the memory -## is divided evenly between the workers. -## In the absence of a workerMemory.query specification, the resourced pod memory -## will be split evenly between the workers in each worker pod. -## NB: numWorkersPerPod must be a factor of numWorkers +## The [manager/worker/eclAgent]Resources define the resource limits for each container. +## If numWorkersPerPod is >1 (must be a factor of numWorkers). +## NB: Each worker corresponds to a container, that will be resourced according to +## workerResources, meaning that if numWorkersPerPod>1, N * workerResources.cpu, +## N * workerResources.memory etc., will be required in total for the pod. +## +## By default the available Thor memory will be based on the resourced container memory. +## This can be overriden by setting [worker/manager]Memory.query and +## [worker/manager]Memory.thirdParty. thor: - name: thor prefix: thor diff --git a/thorlcr/master/thmastermain.cpp b/thorlcr/master/thmastermain.cpp index 72eb2a961f7..30e1cd4c39b 100644 --- a/thorlcr/master/thmastermain.cpp +++ b/thorlcr/master/thmastermain.cpp @@ -814,8 +814,6 @@ int main( int argc, const char *argv[] ) if (!managerMemory->hasProp("@maxMemPercentage")) managerMemory->setPropReal("@maxMemPercentage", localThor ? 25.0 : defaultPctSysMemForRoxie); } - // NB: if (cloud - numWorkersPerPod) or (bare-metal - slavesPerNode) is specified - // the percentage will be split based on numWorkersPerPod or slavesPerNode (see if (numWorkersPerPodOrNode > 1) code below) } workerMemory->setPropInt("@total", gmemSize); @@ -970,7 +968,6 @@ int main( int argc, const char *argv[] ) kjServiceMpTag = allocateClusterMPTag(); unsigned numWorkers = 0; - unsigned numWorkersPerPodOrNode = 1; // pod in cloud, node in bare-metal bool doWorkerRegistration = false; if (isContainerized()) { @@ -1009,7 +1006,6 @@ int main( int argc, const char *argv[] ) Owned workunit = &wuRead->lock(); addTimeStamp(workunit, wfid, graphName, StWhenK8sStarted); } - numWorkersPerPodOrNode = numWorkersPerPod; cloudJobName.appendf("%s-%s", workunit, graphName); @@ -1033,17 +1029,18 @@ int main( int argc, const char *argv[] ) unsigned localThorPortInc = globals->getPropInt("@localThorPortInc", DEFAULT_SLAVEPORTINC); unsigned slaveBasePort = globals->getPropInt("@slaveport", DEFAULT_THORSLAVEPORT); Owned rawGroup = getClusterNodeGroup(thorname, "ThorCluster"); - numWorkersPerPodOrNode = globals->getPropInt("@slavesPerNode", 1); - setClusterGroup(queryMyNode(), rawGroup, numWorkersPerPodOrNode, channelsPerWorker, slaveBasePort, localThorPortInc); + unsigned numWorkersPerNode = globals->getPropInt("@slavesPerNode", 1); + setClusterGroup(queryMyNode(), rawGroup, numWorkersPerNode, channelsPerWorker, slaveBasePort, localThorPortInc); numWorkers = queryNodeClusterWidth(); doWorkerRegistration = true; - } - if (numWorkersPerPodOrNode > 1) - { - // NB: maxMemPercentage only be set when memory amounts have not explicily been defined (e.g. globalMemorySize) - double pct = workerMemory->getPropReal("@maxMemPercentage"); - if (pct) - workerMemory->setPropReal("@maxMemPercentage", pct / numWorkersPerPodOrNode); + if (numWorkersPerNode > 1) + { + // Split memory based on numWorkersPerNode + // NB: maxMemPercentage only set when memory amounts have not explicily been defined (e.g. globalMemorySize) + double pct = workerMemory->getPropReal("@maxMemPercentage"); + if (pct) + workerMemory->setPropReal("@maxMemPercentage", pct / numWorkersPerNode); + } } if (doWorkerRegistration && registry->connect(numWorkers))