From f4c7ce81c395936e4dc2eecb3e8ce3d7741b70c1 Mon Sep 17 00:00:00 2001 From: Mark Kelly Date: Mon, 31 Jul 2023 15:46:48 -0400 Subject: [PATCH 01/23] HPCC-29990 Getaddrinfo call set hints.ai_family to AF_INET unless IP6preferred Signed-off-by: Mark Kelly --- system/jlib/jsocket.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/system/jlib/jsocket.cpp b/system/jlib/jsocket.cpp index 62cdfb7449c..768e2e219ec 100644 --- a/system/jlib/jsocket.cpp +++ b/system/jlib/jsocket.cpp @@ -3530,6 +3530,9 @@ static bool lookupHostAddress(const char *name,unsigned *netaddr) struct addrinfo *addrInfo = NULL; for (;;) { memset(&hints,0,sizeof(hints)); + // dont wait for both A and AAAA records ... + if (IP4only || (!IP6preferred)) + hints.ai_family = AF_INET; int ret = getaddrinfo(name, NULL , &hints, &addrInfo); if (!ret) break; From a9ed1e1838c1c7f90c05c8cdf1bb00bc6b25f62f Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 1 Aug 2023 12:01:53 +0100 Subject: [PATCH 02/23] HPCC-30016 Reduce contention in HalfKeyedJoin::indexReadAllocator Signed-off-by: Gavin Halliday --- common/thorhelper/roxierow.cpp | 10 ++++++++++ ecl/hqlcpp/hqlcppds.cpp | 1 + roxie/ccd/ccdserver.cpp | 5 ++++- rtl/include/eclhelper.hpp | 1 + 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/common/thorhelper/roxierow.cpp b/common/thorhelper/roxierow.cpp index cd599eda04d..48d297361be 100644 --- a/common/thorhelper/roxierow.cpp +++ b/common/thorhelper/roxierow.cpp @@ -323,6 +323,11 @@ class RoxieEngineFixedRowAllocator : public RoxieEngineRowAllocatorBase heap->releaseAllRows(); } + virtual void emptyCache() override + { + heap->emptyCache(); + } + protected: Owned heap; }; @@ -388,6 +393,11 @@ class RoxieEngineVariableRowAllocator : public RoxieEngineRowAllocatorBase throwUnexpected(); } + virtual void emptyCache() override + { + //Variable length rows do not support blocked/caching + } + protected: void * doCreateRow(size32_t initialSize, size32_t & allocatedSize) { diff --git a/ecl/hqlcpp/hqlcppds.cpp b/ecl/hqlcpp/hqlcppds.cpp index a36c80363be..3423683c122 100644 --- a/ecl/hqlcpp/hqlcppds.cpp +++ b/ecl/hqlcpp/hqlcppds.cpp @@ -3120,6 +3120,7 @@ class EclccEngineRowAllocator : public CInterfaceOf virtual IEngineRowAllocator *createChildRowAllocator(const RtlTypeInfo *type) override { throwUnexpected(); } virtual void gatherStats(CRuntimeStatisticCollection & stats) override {} virtual void releaseAllRows() override { throwUnexpected(); } + virtual void emptyCache() override { } }; //Use a (constant) transform to map selectors of the form queryActiveTableSelector().field diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index b960a9296fa..f283e666113 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -26895,7 +26895,8 @@ class CRoxieServerHalfKeyedJoinActivity : public CRoxieServerKeyedJoinBase virtual void onCreate(IHThorArg *_colocalParent) { CRoxieServerKeyedJoinBase::onCreate(_colocalParent); - indexReadAllocator.setown(createRowAllocator(indexReadMeta)); + //The index read allocator->allocate() is only ever called from a single thread => we can allocate rows in blocks + indexReadAllocator.setown(createRowAllocatorEx(indexReadMeta, roxiemem::RHFblocked)); IOutputMetaData *joinFieldsMeta = helper.queryJoinFieldsRecordSize(); joinPrefixedMeta.setown(new CPrefixedOutputMeta(KEYEDJOIN_RECORD_SIZE(0), joinFieldsMeta)); // MORE - not sure if we really need this @@ -26935,6 +26936,8 @@ class CRoxieServerHalfKeyedJoinActivity : public CRoxieServerKeyedJoinBase virtual void reset() { CRoxieServerKeyedJoinBase::reset(); + if (indexReadAllocator) + indexReadAllocator->emptyCache(); if (varFileInfo) { keySet.clear(); diff --git a/rtl/include/eclhelper.hpp b/rtl/include/eclhelper.hpp index f404fea264f..0eaf18f8599 100644 --- a/rtl/include/eclhelper.hpp +++ b/rtl/include/eclhelper.hpp @@ -283,6 +283,7 @@ interface IEngineRowAllocator : extends IInterface virtual void gatherStats(CRuntimeStatisticCollection & stats) = 0; virtual void releaseAllRows() = 0; // Only valid on unique allocators, use with extreme care + virtual void emptyCache() = 0; // Should be called on all blocked allocators. }; interface IRowSerializerTarget From bad0cb39892ab68800b658548b3106592a771da3 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 26 Jul 2023 11:54:01 +0100 Subject: [PATCH 03/23] HPCC-29840 UDP lost packets can lead to a 5s delay Signed-off-by: Richard Chapman --- helm/hpcc/values.schema.json | 6 ++++++ roxie/ccd/ccd.hpp | 1 + roxie/ccd/ccdmain.cpp | 2 ++ roxie/ccd/ccdserver.cpp | 2 ++ 4 files changed, 11 insertions(+) diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index ec1cf9bdc0f..59ed4fd725f 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -1761,6 +1761,12 @@ "minimum": 0, "description": "Minimum number of remote files to keep open" }, + "packetAcknowledgeTimeout": { + "type": "integer", + "default": 100, + "minimum": 10, + "description": "Maximum time to wait for agent query packets to be acknowledged before resending" + }, "parallelQueryLoadThreads": { "type": "integer", "default": 0, diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index 9924e16789c..bc75e91a450 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -309,6 +309,7 @@ extern MapStringTo *preferredClusters; extern StringArray allQuerySetNames; extern bool acknowledgeAllRequests; +extern unsigned packetAcknowledgeTimeout; extern bool alwaysTrustFormatCrcs; extern bool allFilesDynamic; extern bool lockSuperFiles; diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index 08c14ec8d75..04f3b56cacd 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -68,6 +68,7 @@ unsigned numAgentThreads = 30; bool prestartAgentThreads = false; unsigned numRequestArrayThreads = 5; bool acknowledgeAllRequests = true; +unsigned packetAcknowledgeTimeout = 100; unsigned headRegionSize; unsigned ccdMulticastPort; bool enableHeartBeat = true; @@ -916,6 +917,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) } acknowledgeAllRequests = topology->getPropBool("@acknowledgeAllRequests", acknowledgeAllRequests); + packetAcknowledgeTimeout = topology->getPropInt("@packetAcknowledgeTimeout", packetAcknowledgeTimeout); headRegionSize = topology->getPropInt("@headRegionSize", 50); ccdMulticastPort = topology->getPropInt("@multicastPort", CCD_MULTICAST_PORT); statsExpiryTime = topology->getPropInt("@statsExpiryTime", 3600); diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index fc209b1fda1..817f9a5b5ce 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -4938,6 +4938,8 @@ class CRemoteResultAdaptor : implements IEngineRowStream, implements IFinalRoxie unsigned checkInterval = activity.queryContext()->checkInterval(); if (checkInterval > timeout) checkInterval = timeout; + if (acknowledgeAllRequests && checkInterval > packetAcknowledgeTimeout) + checkInterval = packetAcknowledgeTimeout; unsigned lastActivity = msTick(); for (;;) { From 5126124ee88387a51ed6447c2853f7055cb0cae1 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 2 Aug 2023 13:51:00 +0100 Subject: [PATCH 04/23] HPCC-29883 Reduce scope of cs in DataBuffer allocator Signed-off-by: Gavin Halliday --- roxie/roxiemem/roxiemem.cpp | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/roxie/roxiemem/roxiemem.cpp b/roxie/roxiemem/roxiemem.cpp index 74b1b29ae4d..6488bd77f16 100644 --- a/roxie/roxiemem/roxiemem.cpp +++ b/roxie/roxiemem/roxiemem.cpp @@ -6644,7 +6644,7 @@ class CDataBufferManager : implements IDataBufferManager, public CInterface DataBuffer *allocate() { - CriticalBlock b(crit); + CLeavableCriticalBlock block(crit); if (memTraceLevel >= 5) DBGLOG("RoxieMemMgr: CDataBufferManager::allocate() curBlock=%p nextAddr=%p:%x", curBlock, nextBase, nextOffset); @@ -6681,8 +6681,12 @@ class CDataBufferManager : implements IDataBufferManager, public CInterface memsize_t nextHeadId = createDataId(nextFree, nextSeqFromDataId(curHeadId)); if (likely(bottom->freeHeadId.compare_exchange_weak(curHeadId, nextHeadId, std::memory_order_acq_rel))) { - dataBuffersActive.fetch_add(1); bottom->Link(); + + //Leave the critical section before initialising the return result + block.leave(); + + dataBuffersActive.fetch_add(1); curFree->nextDataId = 0; curFree->changeState(DBState::freed, DBState::unowned, __func__); // sanity check - will be overriden by the following new if (memTraceLevel >= 4) @@ -6694,19 +6698,15 @@ class CDataBufferManager : implements IDataBufferManager, public CInterface if (nextOffset < HEAP_ALIGNMENT_SIZE) // Is there any space in the current block (it must be a whole block) { - dataBuffersActive.fetch_add(1); curBlock->Link(); - DataBuffer *x = ::new(nextBase+nextOffset) DataBuffer(); + char * result = nextBase+nextOffset; nextOffset += DATA_ALIGNMENT_SIZE; - if (nextOffset == HEAP_ALIGNMENT_SIZE) - { - // MORE: May want to delete this "if" logic !! - // and let it be handled in the similar logic of "else" part below. - curBlock->Release(); - curBlock = NULL; - nextBase = NULL; - //nextOffset = 0 - not needed since only used if curBlock is set - } + + //Leave the critical section before initialising the return result + block.leave(); + + dataBuffersActive.fetch_add(1); + DataBuffer *x = ::new(result) DataBuffer(); if (memTraceLevel >= 4) DBGLOG("RoxieMemMgr: CDataBufferManager::allocate() allocated DataBuffer - addr=%p", x); return x; @@ -6749,8 +6749,12 @@ class CDataBufferManager : implements IDataBufferManager, public CInterface memsize_t nextHeadId = createDataId(nextFree, nextSeqFromDataId(curHeadId)); if (likely(finger->freeHeadId.compare_exchange_weak(curHeadId, nextHeadId, std::memory_order_acq_rel))) { - dataBuffersActive.fetch_add(1); finger->Link(); + + //Leave the critical section before initialising the return result + block.leave(); + + dataBuffersActive.fetch_add(1); curFree->nextDataId = 0; curFree->changeState(DBState::freed, DBState::unowned, __func__); // sanity check - will be overriden by the following new if (memTraceLevel >= 4) From e3e40dea7b8e192252e70d3bcdc6558d74e61504 Mon Sep 17 00:00:00 2001 From: wangkx Date: Wed, 28 Jun 2023 10:59:36 -0400 Subject: [PATCH 05/23] HPCC-29246 Check Plane scope in DFU server when spraying/despraying Revise based on review: 1. Merge the getPlaneFilePermissions() into the checkPlaneFilePermissions(). 2. Add option failOverToLegacyPhysicalPerms. 3. If no plane name, check the DropZoneRestrictionEnabled. 4. Check read permission only for read action 5. Change the ensureFilePermissions 6. Call the ensureFilePermissions in checkForeignFilePermissions 7. Check isAbsolute before calling getRelativePath 8. Allow empty relative path in setPlaneExternal() Signed-off-by: wangkx --- dali/base/dautils.cpp | 7 ++-- dali/dfu/dfurun.cpp | 83 ++++++++++++++++++++++++++++++------- esp/smc/SMCLib/TpCommon.cpp | 16 +++---- system/jlib/jfile.cpp | 18 ++++++++ system/jlib/jfile.hpp | 1 + 5 files changed, 95 insertions(+), 30 deletions(-) diff --git a/dali/base/dautils.cpp b/dali/base/dautils.cpp index 78c3463bf10..0f08397d834 100644 --- a/dali/base/dautils.cpp +++ b/dali/base/dautils.cpp @@ -1138,13 +1138,12 @@ static void convertPosixPathToLfn(StringBuffer &str,const char *path) void CDfsLogicalFileName::setPlaneExternal(const char *plane,const char *path) { - if (isEmptyString(path)) - return; - if (isPathSepChar(path[0])&&(path[0]==path[1])) + if (!isEmptyString(path)&&isPathSepChar(path[0])&&(path[0]==path[1])) throw makeStringExceptionV(-1,"Invalid path %s.",path); StringBuffer str(PLANE_SCOPE "::"); str.append(plane); - convertPosixPathToLfn(str,path); + if (!isEmptyString(path)) + convertPosixPathToLfn(str,path); set(str.str()); } diff --git a/dali/dfu/dfurun.cpp b/dali/dfu/dfurun.cpp index 27cdd17f0ad..0fe8adcb343 100644 --- a/dali/dfu/dfurun.cpp +++ b/dali/dfu/dfurun.cpp @@ -533,6 +533,24 @@ class CDFUengine: public CInterface, implements IDFUengine } } + StringBuffer & getFDescName(IFileDescriptor * fd, StringBuffer & result) + { + fd->getTraceName(result); + elideString(result, 255); + return result; + } + + void ensureFilePermissions(const char * fileName, SecAccessFlags perm, bool write) + { + if ((write && !HASWRITEPERMISSION(perm)) || (!write && !HASREADPERMISSION(perm))) + { + if (write) + throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for physical file(s): %s", fileName); + else + throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for physical file(s): %s", fileName); + } + } + Linked config; Owned eventpusher; IArrayOf listeners; @@ -595,16 +613,8 @@ class CDFUengine: public CInterface, implements IDFUengine auditflags |= DALI_LDAP_WRITE_WANTED; SecAccessFlags perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags); - if ((write && !HASWRITEPERMISSION(perm)) || (!write && !HASREADPERMISSION(perm))) - { - StringBuffer traceName; - fd->getTraceName(traceName); - elideString(traceName, 255); - if (write) - throw makeStringExceptionV(DFSERR_CreateAccessDenied, "Create permission denied for physical file(s): %s",traceName.str()); - else - throw makeStringExceptionV(DFSERR_LookupAccessDenied, "Lookup permission denied for physical file(s): %s",traceName.str()); - } + StringBuffer name; + ensureFilePermissions(getFDescName(fd,name),perm,write); } void checkForeignFilePermissions(IConstDFUfileSpec *fSpec,IFileDescriptor *fd,IUserDescriptor *user,bool write) @@ -638,14 +648,55 @@ class CDFUengine: public CInterface, implements IDFUengine bool checkLegacyPhysicalPerms = getGlobalConfigSP()->getPropBool("expert/@failOverToLegacyPhysicalPerms",!isContainerized()); if (((!write&&!HASREADPERMISSION(perm)) || (write&&!HASWRITEPERMISSION(perm))) && checkLegacyPhysicalPerms) perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags); + ensureFilePermissions(logicalName,perm,write); + } + void checkPlaneFilePermissions(IFileDescriptor *fd,IUserDescriptor *user,bool write) + { + //This function checks the scope permissions for a file or files that reside in a single directory on a single plane. + //The IFileDescriptor is used to discover the plane and directory. + //If the plane is not present, it implies that it is a bare-metal system and useDropZoneRestriction is off, and there + //is no matching dropzone in the environment. If this is the case, or the plane permissions are not found + //and @failOverToLegacyPhysicalPerms is configured, then the legacy physical file permissions will be checked. + unsigned auditflags = (DALI_LDAP_AUDIT_REPORT|DALI_LDAP_READ_WANTED); if (write) + auditflags |= DALI_LDAP_WRITE_WANTED; + + SecAccessFlags perm; + bool checkLegacyPhysicalPerms = getGlobalConfigSP()->getPropBool("expert/@failOverToLegacyPhysicalPerms",!isContainerized()); + IClusterInfo *iClusterInfo = fd->queryClusterNum(0); + const char *planeName = iClusterInfo->queryGroupName(); + if (!isEmptyString(planeName)) { - if (!HASWRITEPERMISSION(perm)) - throw makeStringExceptionV(DFSERR_CreateAccessDenied,"Create permission denied for foreign file: %s",logicalName.str()); + const char *dir = fd->queryDefaultDir(); + if (isEmptyString(dir)) + throw makeStringException(-1,"Empty default directory."); + + Owned dropZonePlane = getDropZonePlane(planeName); + if (!dropZonePlane) + throw makeStringExceptionV(-1,"DropZone %s not found.",planeName); + const char *relativePath = getRelativePath(dir,dropZonePlane->queryProp("@prefix")); + if (nullptr == relativePath) + throw makeStringExceptionV(-1,"Invalid DropZone directory %s.",dir); + + perm = queryDistributedFileDirectory().getDropZoneScopePermissions(planeName,relativePath,user,auditflags); + if (((!write&&!HASREADPERMISSION(perm))||(write&&!HASWRITEPERMISSION(perm)))&&checkLegacyPhysicalPerms) + perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags); + } + else + { +#ifndef _CONTAINERIZED + Owned factory = getEnvironmentFactory(true); + Owned env = factory->openEnvironment(); + if (env->isDropZoneRestrictionEnabled()||!checkLegacyPhysicalPerms) + throw makeStringException(-1,"Empty plane name."); + perm = queryDistributedFileDirectory().getFDescPermissions(fd,user,auditflags); +#else + throw makeStringException(-1,"Unexpected empty plane name."); // should never be the case in containerized setups +#endif } - else if (!HASREADPERMISSION(perm)) - throw makeStringExceptionV(DFSERR_LookupAccessDenied,"Lookup permission denied for foreign file: %s",logicalName.str()); + StringBuffer name; + ensureFilePermissions(getFDescName(fd,name),perm,write); } void monitorCycle(bool &cancelling) @@ -1741,7 +1792,7 @@ class CDFUengine: public CInterface, implements IDFUengine if (!replicating) { runningconn.setown(setRunning(runningpath.str())); Owned fdesc = source->getFileDescriptor(); - checkPhysicalFilePermissions(fdesc,userdesc,false); + checkPlaneFilePermissions(fdesc,userdesc,false); checkSourceTarget(fdesc); bool needrep = options->getReplicate(); ClusterPartDiskMapSpec mspec; @@ -1777,7 +1828,7 @@ class CDFUengine: public CInterface, implements IDFUengine { runningconn.setown(setRunning(runningpath.str())); Owned fdesc = destination->getFileDescriptor(iskey); - checkPhysicalFilePermissions(fdesc,userdesc,true); + checkPlaneFilePermissions(fdesc,userdesc,true); checkSourceTarget(fdesc); fsys.exportFile(srcFile, fdesc, recovery, recoveryconn, filter, opttree, &feedback, &abortnotify, dfuwuid); if (!abortnotify.abortRequested()) { diff --git a/esp/smc/SMCLib/TpCommon.cpp b/esp/smc/SMCLib/TpCommon.cpp index a5e8612b622..195c2f6e909 100644 --- a/esp/smc/SMCLib/TpCommon.cpp +++ b/esp/smc/SMCLib/TpCommon.cpp @@ -183,21 +183,17 @@ static SecAccessFlags getDropZoneScopePermissions(IEspContext& context, const IP throw makeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "getDropZoneScopePermissions(): DropZone path must be specified."); //If the dropZonePath is an absolute path, change it to a relative path. - StringBuffer s; - const char* prefix = dropZone->queryProp("@prefix"); - const char* name = dropZone->queryProp("@name"); - if (hasPrefix(dropZonePath, prefix, true)) + if (isAbsolutePath(dropZonePath)) { - const char* p = dropZonePath + strlen(prefix); - if (!*p || !isPathSepChar(p[0])) - addPathSepChar(s); - s.append(p); - dropZonePath = s.str(); + const char* relativePath = getRelativePath(dropZonePath, dropZone->queryProp("@prefix")); + if (nullptr == relativePath) + throw makeStringExceptionV(-1, "Invalid DropZone path %s.", dropZonePath); + dropZonePath = relativePath; } Owned userDesc = createUserDescriptor(); userDesc->set(context.queryUserId(), context.queryPassword(), context.querySignature()); - return queryDistributedFileDirectory().getDropZoneScopePermissions(name, dropZonePath, userDesc); + return queryDistributedFileDirectory().getDropZoneScopePermissions(dropZone->queryProp("@name"), dropZonePath, userDesc); } extern TPWRAPPER_API SecAccessFlags getDZPathScopePermissions(IEspContext& context, const char* dropZoneName, const char* dropZonePath, const char* dropZoneHost) diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index 694aec0f4fb..507874517b8 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -5706,6 +5706,24 @@ const char *splitRelativePath(const char *full,const char *basedir,StringBuffer return t; } +const char *getRelativePath(const char *path,const char *leadingPath) +{ + size_t pathLen = strlen(path); + size_t leadingLen = strlen(leadingPath); + if ((pathLen==leadingLen-1)&&isPathSepChar(leadingPath[leadingLen-1])) + --leadingLen; + if (0 == strncmp(path,leadingPath,leadingLen)) + { + const char *rel = path + leadingLen; + if ('\0' == *rel) + return rel; + if (isPathSepChar(*rel)) + return rel+1; + return rel; + } + return nullptr; +} + const char *splitDirMultiTail(const char *multipath,StringBuffer &dir,StringBuffer &tail) { // the first directory is the significant one diff --git a/system/jlib/jfile.hpp b/system/jlib/jfile.hpp index 6ab5b339372..bbd5f33087f 100644 --- a/system/jlib/jfile.hpp +++ b/system/jlib/jfile.hpp @@ -606,6 +606,7 @@ extern jlib_decl StringBuffer &makeAbsolutePath(StringBuffer &relpath,bool mustE extern jlib_decl StringBuffer &makeAbsolutePath(const char *relpath, const char *basedir, StringBuffer &out); extern jlib_decl StringBuffer &makePathUniversal(const char *path, StringBuffer &out); extern jlib_decl const char *splitRelativePath(const char *full,const char *basedir,StringBuffer &reldir); // removes basedir if matches, returns tail and relative dir +extern jlib_decl const char *getRelativePath(const char *path, const char *leadingPath); extern jlib_decl const char *splitDirMultiTail(const char *multipath,StringBuffer &dir,StringBuffer &tail); extern jlib_decl StringBuffer &mergeDirMultiTail(const char *dir,const char *tail, StringBuffer &multipath); extern jlib_decl StringBuffer &removeRelativeMultiPath(const char *full,const char *dir,StringBuffer &reltail); // removes dir if matches, returns relative multipath From 520af37183f87e8dede962aaaa404bf27efdc375 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 3 Aug 2023 17:22:53 +0100 Subject: [PATCH 06/23] Split off 8.12.42 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 798c4c1142c..57a338751da 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 8.12.41-closedown0 +version: 8.12.43-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 8.12.41-closedown0 +appVersion: 8.12.43-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index db7472db872..d56d09489b6 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1225,7 +1225,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index fd51fff091e..86bde51460a 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 98cf91b8323..19fd535738d 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index eaa5a1caec7..be04a299741 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index b11385df41b..dde133ec16a 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 3c2f75738c0..83dccabf8c9 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index f87a3708951..97f1bde3637 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index f8ae5a8cc45..c0ed3e75611 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 4ae8e1ee774..57a6f0ace1d 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 9d3241d4d9c..8b371e05ecb 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index df478896c4b..4113a88f429 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 75fcc4e791f..8240b8d6a4f 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -149,7 +149,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -353,7 +353,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -418,7 +418,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 8.12.41-closedown0 + helmVersion: 8.12.43-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index dd859b43e7b..e6da983e169 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 8 ) set ( HPCC_MINOR 12 ) -set ( HPCC_POINT 41 ) +set ( HPCC_POINT 43 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From fc1a7545ee1682ef2aa7948ad66875cdfc10c309 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 3 Aug 2023 17:26:56 +0100 Subject: [PATCH 07/23] Split off 9.0.32 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index d170737b183..7946b81023c 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.0.31-closedown0 +version: 9.0.33-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.0.31-closedown0 +appVersion: 9.0.33-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 918a0c746fb..4298299ce50 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1299,7 +1299,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index eb43b0eefbb..28c6f487789 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 662275c7510..de39c898b72 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 9f05f2653d3..44d64b55f6e 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index ce216a6a320..d93ef473fe7 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 5106adecfc1..8bf72c44e94 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index df5f8cc7bd9..0d5280792c8 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 81c55ae977f..8adc1ff95e5 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 58a20f970e4..70b33827755 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index c8e815a9a59..a79a4252282 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 64c34d7bd29..914430575c8 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index ae5a5d1ba8d..6ba8cdbd30e 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.31-closedown0 + helmVersion: 9.0.33-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 77ef56ae9bd..1555862be95 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 0 ) -set ( HPCC_POINT 31 ) +set ( HPCC_POINT 33 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From 7fe7615d1045b6d0cc63d1a59c382c38a3b1b12a Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 3 Aug 2023 17:31:04 +0100 Subject: [PATCH 08/23] Split off 9.2.10 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 5409ddbd1d2..e4cebd9162a 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.9-closedown0 +version: 9.2.11-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.9-closedown0 +appVersion: 9.2.11-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 8d8acbda03a..01e9e706cbd 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1320,7 +1320,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index e0c42ed041a..7984425d5b4 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 6a636eff8eb..60392897579 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 284f8343ec5..ed6cb8bd026 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index fcf293b9293..d50b5a10f22 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 4376092f02e..b6d9970c61d 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index d592116cd57..23c09c80931 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 5888eb55de1..083ef94a8d3 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index b43f11a6dbc..2952aa2d0f3 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 27a2110eb4e..cfe9c6fe93c 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index ce29da6ab13..edc1a4753d0 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 520a566c20a..32788a9472e 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.9-closedown0 + helmVersion: 9.2.11-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 6f00e6a2a63..74081b22944 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 9 ) +set ( HPCC_POINT 11 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From 7bcd9c44e9dc75889dd2fa1acbc658f7257d84a7 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 3 Aug 2023 19:47:57 +0100 Subject: [PATCH 09/23] HPCC-29301 Guard against potential (theoretical) use of null pointer Signed-off-by: Gavin Halliday --- system/jhtree/jhinplace.cpp | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/system/jhtree/jhinplace.cpp b/system/jhtree/jhinplace.cpp index f35a89a87db..38c9e15e6b9 100644 --- a/system/jhtree/jhinplace.cpp +++ b/system/jhtree/jhinplace.cpp @@ -2507,21 +2507,25 @@ void CInplaceLeafWriteNode::write(IFileIOStream *out, CRC32 *crc) InplaceIndexCompressor::InplaceIndexCompressor(size32_t keyedSize, const CKeyHdr * keyHdr, IHThorIndexWriteArg * helper, const char * _compressionName) : compressionName(_compressionName) { - IOutputMetaData * format = helper->queryDiskRecordSize(); - if (format) + if (helper) { - //Create a representation of the null row - potentially used for the new compression algorithm - byte * nullRow = new byte[keyedSize]; - RtlStaticRowBuilder rowBuilder(nullRow, keyedSize); - - auto & meta = format->queryRecordAccessor(true); - size32_t offset = 0; - for (unsigned idx = 0; idx < meta.getNumFields() && offset < keyedSize; idx++) + //Nothing currently relies on the null row being created.... + IOutputMetaData * format = helper->queryDiskRecordSize(); + if (format) { - const RtlFieldInfo *field = meta.queryField(idx); - offset = field->type->buildNull(rowBuilder, offset, field); + //Create a representation of the null row - potentially used for the new compression algorithm + byte * nullRow = new byte[keyedSize]; + RtlStaticRowBuilder rowBuilder(nullRow, keyedSize); + + auto & meta = format->queryRecordAccessor(true); + size32_t offset = 0; + for (unsigned idx = 0; idx < meta.getNumFields() && offset < keyedSize; idx++) + { + const RtlFieldInfo *field = meta.queryField(idx); + offset = field->type->buildNull(rowBuilder, offset, field); + } + ctx.nullRow = nullRow; } - ctx.nullRow = nullRow; } //Process any options. Currently supports the compression mode From fe405083437e7d6002db796934e10cdbc1f06aba Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 3 Aug 2023 20:13:23 +0100 Subject: [PATCH 10/23] HPCC-27304 Remove uninitialised variable false positive Signed-off-by: Gavin Halliday --- common/deftype/defvalue.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/deftype/defvalue.cpp b/common/deftype/defvalue.cpp index 344511b8307..47fd7ccc71d 100644 --- a/common/deftype/defvalue.cpp +++ b/common/deftype/defvalue.cpp @@ -2033,6 +2033,7 @@ void RealValue::toMem(void *target) RealUnion u; size32_t size = type->getSize(); + u.r8 = 0; switch (size) { case 4: @@ -2050,6 +2051,7 @@ unsigned RealValue::getHash(unsigned initval) RealUnion u; size32_t size = type->getSize(); + u.r8 = 0; switch (size) { case 4: From e11b77f127877c29cafbe133b3d8b01001f2fb72 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 3 Aug 2023 20:17:34 +0100 Subject: [PATCH 11/23] HPCC-26703 Ensure members of KeyPatchParams are initialised Signed-off-by: Gavin Halliday --- tools/keydiff/patchmain.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/keydiff/patchmain.cpp b/tools/keydiff/patchmain.cpp index b6db0c1df11..a7db02b960a 100644 --- a/tools/keydiff/patchmain.cpp +++ b/tools/keydiff/patchmain.cpp @@ -92,15 +92,15 @@ class KeyPatchParams StringBuffer oldIndex; StringBuffer newIndex; StringBuffer newTLK; - bool overwrite; - KeyPatchMode mode; - bool xmitTLK; - bool recvTLK; + bool overwrite = false; + KeyPatchMode mode = KEYPATCH_implicit; + bool xmitTLK = false; + bool recvTLK = false; SocketEndpoint xmitEp; - unsigned recvNum; - unsigned tlkPort; - bool ignoreTLK; - offset_t progressFrequency; + unsigned recvNum = 0; + unsigned tlkPort = 0; + bool ignoreTLK = false; + offset_t progressFrequency = 0; }; void getParams(unsigned argc, char * const * argv, KeyPatchParams & params) From 8a81661d54efe3701f512bff02362392e2d3d390 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 3 Aug 2023 20:32:29 +0100 Subject: [PATCH 12/23] HPCC-27317 Add brackets to suppress precedence warning Signed-off-by: Gavin Halliday --- rtl/nbcd/nbcd.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rtl/nbcd/nbcd.cpp b/rtl/nbcd/nbcd.cpp index 291ae4016f3..92081b27a37 100644 --- a/rtl/nbcd/nbcd.cpp +++ b/rtl/nbcd/nbcd.cpp @@ -116,14 +116,14 @@ int Decimal::compare(const Decimal & other) const return negative ? -1 : +1; if (hi1 != hi2) - return (hi1 > hi2) ^ negative ? +1 : -1; + return ((hi1 > hi2) ^ negative) ? +1 : -1; int limit = lo1 < lo2 ? lo2 : lo1; for (;hi1 >= limit; hi1--) { int diff = digits[hi1] - other.digits[hi1]; if (diff != 0) - return (diff > 0) ^ negative ? +1 : -1; + return ((diff > 0) ^ negative) ? +1 : -1; } if (lo1 == lo2) From 965a9aef184796ad4816c88fce52bec983bf05de Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 4 Aug 2023 12:39:04 +0100 Subject: [PATCH 13/23] HPCC-30020 Support legacy DropZones without ServerList Signed-off-by: Jake Smith --- dali/base/dadfs.cpp | 91 +++++++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 37 deletions(-) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 03a98f0fc1c..351e2a2a765 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -10063,7 +10063,31 @@ class CInitGroups addHostsToIPTFunc(); return cluster.getClear(); } - + const char *getHostFromClusterEntry(const IPropertyTree &node, const char *clusterName) + { + const char *computer = node.queryProp("@computer"); + if (!isEmptyString(computer)) + { + auto it = machineMap.find(computer); + if (it == machineMap.end()) + { + OERRLOG("Cannot construct %s, computer name %s not found\n", clusterName, computer); + return nullptr; + } + return it->second.c_str(); + } + else + { + const char *host = node.queryProp("@netAddress"); + if (isEmptyString(host)) + { + OERRLOG("Cannot construct %s, missing computer spec on node\n", clusterName); + return nullptr; + } + else + return host; + } + } IPropertyTree *createClusterGroupFromEnvCluster(GroupType groupType, const IPropertyTree &cluster, const char *dir, bool realCluster, bool expand) { const char *processName=nullptr; @@ -10086,50 +10110,43 @@ class CInitGroups } std::vector hosts; Owned nodes = cluster.getElements(processName); - ForEach(*nodes) + if (nodes->first()) { - IPropertyTree &node = nodes->query(); - const char *host = nullptr; - if (grp_dropzone == groupType) - host = node.queryProp("@server"); - else + do { - const char *computer = node.queryProp("@computer"); - if (!isEmptyString(computer)) - { - auto it = machineMap.find(computer); - if (it == machineMap.end()) - { - OERRLOG("Cannot construct %s, computer name %s not found\n", cluster.queryProp("@name"), computer); - return nullptr; - } - host = it->second.c_str(); - } + IPropertyTree &node = nodes->query(); + const char *host = nullptr; + if (grp_dropzone == groupType) + host = node.queryProp("@server"); else + host = getHostFromClusterEntry(node, cluster.queryProp("@name")); + switch (groupType) { - host = node.queryProp("@netAddress"); - if (isEmptyString(host)) - { - OERRLOG("Cannot construct %s, missing computer spec on node\n", cluster.queryProp("@name")); - return nullptr; - } + case grp_roxie: + // Redundant copies are located via the flags. + // Old environments may contain duplicated sever information for multiple ports + if (hosts.end() == std::find(hosts.begin(), hosts.end(), host)) // only add if not already there + hosts.push_back(host); + break; + case grp_thor: + case grp_thorspares: + case grp_dropzone: + hosts.push_back(host); + break; + default: + throwUnexpected(); } } - switch (groupType) + while (nodes->next()); + } + else if (grp_dropzone == groupType) + { + // legacy support for DropZone's without ServerList + if (cluster.hasProp("@computer") || cluster.hasProp("@netAddress")) { - case grp_roxie: - // Redundant copies are located via the flags. - // Old environments may contain duplicated sever information for multiple ports - if (hosts.end() == std::find(hosts.begin(), hosts.end(), host)) // only add if not already there - hosts.push_back(host); - break; - case grp_thor: - case grp_thorspares: - case grp_dropzone: + const char *host = getHostFromClusterEntry(cluster, cluster.queryProp("@name")); + if (!isEmptyString(host)) hosts.push_back(host); - break; - default: - throwUnexpected(); } } if (!hosts.size()) From 2049c0c3d1ac0c00b0d4a48d8ef4301b520390ee Mon Sep 17 00:00:00 2001 From: Mark Kelly Date: Mon, 31 Jul 2023 16:46:12 -0400 Subject: [PATCH 14/23] HPCC-29992 SOAPCALL check timelimit exceeded after call to getaddrinfo Signed-off-by: Mark Kelly --- common/thorhelper/thorsoapcall.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index eade6a2e5c4..3f7f7cd3fe0 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -2234,6 +2234,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo checkTimeLimitExceeded(&remainingMS); Url &connUrl = master->proxyUrlArray.empty() ? url : master->proxyUrlArray.item(0); ep.set(connUrl.host.get(), connUrl.port); + checkTimeLimitExceeded(&remainingMS); // after ep.set which might make a potentially long getaddrinfo lookup ... if (strieq(url.method, "https")) proto = PersistentProtocol::ProtoTLS; bool shouldClose = false; From dd6acfd70ca4699c66886d3cfbb0e8b0b03cce33 Mon Sep 17 00:00:00 2001 From: Jim DeFabia Date: Fri, 4 Aug 2023 14:11:15 -0400 Subject: [PATCH 15/23] HPCC-30042 Fix examples for FAIL, FAILCODE, and FAILMESSAGE Signed-off-by: Jim DeFabia --- .../ECLR_mods/BltInFunc-FAIL.xml | 34 +++++++++++++------ .../ECLR_mods/BltInFunc-FAILCODE.xml | 33 ++++++++++++++---- .../ECLR_mods/BltInFunc-FAILMESSAGE.xml | 32 ++++++++++++++--- 3 files changed, 78 insertions(+), 21 deletions(-) diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-FAIL.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-FAIL.xml index d59bda4643b..2977b74954a 100644 --- a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-FAIL.xml +++ b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-FAIL.xml @@ -77,16 +77,30 @@ Example: - IF(header.version <> doxie.header_version_new, - FAIL('Mismatch -- header.version vs. doxie.header_version_new.')); - -FailedJob := FAIL('ouch, it broke'); -sPeople := SORT(Person,Person.per_first_name); -nUniques := COUNT(DEDUP(sPeople,Person.per_first_name AND - Person.address)) - : FAILURE(FailedJob); -MyRecSet := IF(EXISTS(Person),Person, - FAIL(Person,99,'Person does not exist!!')); + MyRec := RECORD + STRING50 Value1; + UNSIGNED Value2; +END; + +ds := DATASET([{'C',1},{'C',2},{'C',3}, + {'C',4},{'C',5},{'X',1},{'A',1}],MyRec); + +MyRec FailTransform := TRANSFORM + self.value1 := FAILMESSAGE[1..17]; + self.value2 := FAILCODE +END; + +limited1 := LIMIT(ds, 2); +limited2 := LIMIT(ds, 3); +limited3 := LIMIT(ds, 4); + +recovered1 := CATCH(limited1, SKIP); +recovered2 := CATCH(limited2, ONFAIL(FailTransform)); +recovered3 := CATCH(CATCH(limited3, FAIL(1, 'Failed, sorry')), ONFAIL(FailTransform)); + +OUTPUT(recovered1); //empty recordset +OUTPUT(recovered2); // +OUTPUT(recovered3); // See Also: FAILURE, Example: - SPeople := SORT(Person,Person.per_first_name); -nUniques := COUNT(DEDUP(sPeople,Person.per_first_name AND - Person.address)) -:FAILURE(Email.simpleSend(SystemsPersonnel, -SystemsPersonel.email,FAILCODE)); + MyRec := RECORD + STRING50 Value1; + UNSIGNED Value2; +END; - See Also: FAILURE, FAILMESSAGE, SOAPCALL +ds := DATASET([{'C',1},{'C',2},{'C',3}, + {'C',4},{'C',5},{'X',1},{'A',1}],MyRec); + +MyRec FailTransform := TRANSFORM + self.value1 := FAILMESSAGE[1..17]; + self.value2 := FAILCODE +END; + +limited1 := LIMIT(ds, 2); +limited2 := LIMIT(ds, 3); +limited3 := LIMIT(ds, 4); + +recovered1 := CATCH(limited1, SKIP); +recovered2 := CATCH(limited2, ONFAIL(FailTransform)); +recovered3 := CATCH(CATCH(limited3, FAIL(1, 'Failed, sorry')), ONFAIL(FailTransform)); + +OUTPUT(recovered1); //empty recordset +OUTPUT(recovered2); // +OUTPUT(recovered3); // + + See Also: FAILURE, FAILMESSAGE, SOAPCALL diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-FAILMESSAGE.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-FAILMESSAGE.xml index 174ab8c118e..408b6311703 100644 --- a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-FAILMESSAGE.xml +++ b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-FAILMESSAGE.xml @@ -37,10 +37,32 @@ Example: - SPeople := SORT(Person,Person.per_first_name); -nUniques := COUNT(DEDUP(sPeople,Person.per_first_name ANDPerson.address)) -:FAILURE(Email.simpleSend(SystemsPersonnel, - SystemsPersonel.email,FAILMESSAGE)); + MyRec := RECORD + STRING50 Value1; + UNSIGNED Value2; +END; - See Also: RECOVERY, FAILCODE, SOAPCALL +ds := DATASET([{'C',1},{'C',2},{'C',3}, + {'C',4},{'C',5},{'X',1},{'A',1}],MyRec); + +MyRec FailTransform := TRANSFORM + self.value1 := FAILMESSAGE[1..17]; + self.value2 := FAILCODE +END; + +limited1 := LIMIT(ds, 2); +limited2 := LIMIT(ds, 3); +limited3 := LIMIT(ds, 4); + +recovered1 := CATCH(limited1, SKIP); +recovered2 := CATCH(limited2, ONFAIL(FailTransform)); +recovered3 := CATCH(CATCH(limited3, FAIL(1, 'Failed, sorry')), ONFAIL(FailTransform)); + +OUTPUT(recovered1); //empty recordset +OUTPUT(recovered2); // +OUTPUT(recovered3); // + + See Also: RECOVERY, FAILCODE, SOAPCALL From cea551b1e4b2a05f4a19f1580fd975cfe9f80308 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Mon, 7 Aug 2023 16:56:40 +0100 Subject: [PATCH 16/23] HPCC-30045 Add getOptReal to job/graph Signed-off-by: Jake Smith --- thorlcr/graph/thgraph.cpp | 14 ++++++++++++++ thorlcr/graph/thgraph.hpp | 4 ++++ thorlcr/graph/thgraphmaster.cpp | 5 +++++ thorlcr/graph/thgraphmaster.ipp | 8 +++++--- thorlcr/graph/thgraphslave.cpp | 6 ++++++ thorlcr/graph/thgraphslave.hpp | 11 ++++++----- thorlcr/thorutil/thormisc.cpp | 7 +++++++ thorlcr/thorutil/thormisc.hpp | 1 + 8 files changed, 48 insertions(+), 8 deletions(-) diff --git a/thorlcr/graph/thgraph.cpp b/thorlcr/graph/thgraph.cpp index 7c98f60872f..359d432586e 100644 --- a/thorlcr/graph/thgraph.cpp +++ b/thorlcr/graph/thgraph.cpp @@ -473,6 +473,13 @@ __int64 CGraphElementBase::getOptInt64(const char *prop, __int64 defVal) const return queryXGMML().getPropInt64(path.toLowerCase().str(), def); } +double CGraphElementBase::getOptReal(const char *prop, double defVal) const +{ + double def = queryJob().getOptReal(prop, defVal); + VStringBuffer path("hint[@name=\"%s\"]/@value", prop); + return queryXGMML().getPropReal(path.toLowerCase().str(), def); +} + IThorGraphDependencyIterator *CGraphElementBase::getDependsIterator() const { return new ArrayIIteratorOf(dependsOn); @@ -2975,6 +2982,13 @@ __int64 CJobBase::getOptInt64(const char *opt, __int64 dft) return getWorkUnitValueInt(opt, getExpertOptInt64(opt, dft)); } +double CJobBase::getOptReal(const char *opt, double dft) +{ + if (!opt || !*opt) + return dft; // probably error + return getWorkUnitValueReal(opt, getExpertOptReal(opt, dft)); +} + IThorAllocator *CJobBase::getThorAllocator(unsigned channel) { return sharedAllocator.getLink(); diff --git a/thorlcr/graph/thgraph.hpp b/thorlcr/graph/thgraph.hpp index 8dccc1881d2..d2e04ba334c 100644 --- a/thorlcr/graph/thgraph.hpp +++ b/thorlcr/graph/thgraph.hpp @@ -461,6 +461,7 @@ class graph_decl CGraphElementBase : public CInterface, implements IInterface int getOptInt(const char *prop, int defVal=0) const; unsigned getOptUInt(const char *prop, unsigned defVal=0) const { return (unsigned)getOptInt(prop, defVal); } __int64 getOptInt64(const char *prop, __int64 defVal=0) const; + double getOptReal(const char *prop, double defVal) const; unsigned __int64 getOptUInt64(const char *prop, unsigned __int64 defVal=0) const { return (unsigned __int64)getOptInt64(prop, defVal); } void ActPrintLog(const char *format, ...) __attribute__((format(printf, 2, 3))); void ActPrintLog(IException *e, const char *format, ...) __attribute__((format(printf, 3, 4))); @@ -943,6 +944,7 @@ class graph_decl CJobBase : public CInterfaceOf, implements IExcepti virtual __int64 getWorkUnitValueInt(const char *prop, __int64 defVal) const = 0; virtual StringBuffer &getWorkUnitValue(const char *prop, StringBuffer &str) const = 0; virtual bool getWorkUnitValueBool(const char *prop, bool defVal) const = 0; + virtual double getWorkUnitValueReal(const char *prop, double defVal) const = 0; const char *queryWuid() const { return wuid.str(); } const char *queryUser() const { return user.str(); } const char *queryScope() const { return scope.str(); } @@ -962,6 +964,7 @@ class graph_decl CJobBase : public CInterfaceOf, implements IExcepti unsigned getOptUInt(const char *opt, unsigned dft=0) { return (unsigned)getOptInt(opt, dft); } __int64 getOptInt64(const char *opt, __int64 dft=0); unsigned __int64 getOptUInt64(const char *opt, unsigned __int64 dft=0) { return (unsigned __int64)getOptInt64(opt, dft); } + double getOptReal(const char *prop, double defValue=0); IThorAllocator *querySharedAllocator() const { return sharedAllocator; } unsigned getWfid() const { return graphXGMML->getPropInt("@wfid"); } virtual IThorAllocator *getThorAllocator(unsigned channel); @@ -1184,6 +1187,7 @@ class graph_decl CActivityBase : implements CInterfaceOf, im int getOptInt(const char *prop, int defVal=0) const { return container.getOptInt(prop, defVal); } unsigned getOptUInt(const char *prop, unsigned defVal=0) const { return container.getOptUInt(prop, defVal); } __int64 getOptInt64(const char *prop, __int64 defVal=0) const { return container.getOptInt64(prop, defVal); } + double getOptReal(const char *prop, double defVal=0) const { return container.getOptReal(prop, defVal); } unsigned __int64 getOptUInt64(const char *prop, unsigned __int64 defVal=0) const { return container.getOptUInt64(prop, defVal); } }; diff --git a/thorlcr/graph/thgraphmaster.cpp b/thorlcr/graph/thgraphmaster.cpp index 7e5a597c2bb..b2e78c0d995 100644 --- a/thorlcr/graph/thgraphmaster.cpp +++ b/thorlcr/graph/thgraphmaster.cpp @@ -2060,6 +2060,11 @@ bool CJobMaster::getWorkUnitValueBool(const char *prop, bool defVal) const return queryWorkUnit().getDebugValueBool(prop, defVal); } +double CJobMaster::getWorkUnitValueReal(const char *prop, double defVal) const +{ + return queryWorkUnit().getDebugValueReal(prop, defVal); +} + StringBuffer &CJobMaster::getWorkUnitValue(const char *prop, StringBuffer &str) const { SCMStringBuffer scmStr; diff --git a/thorlcr/graph/thgraphmaster.ipp b/thorlcr/graph/thgraphmaster.ipp index a69ff028e87..af60ce2ede6 100644 --- a/thorlcr/graph/thgraphmaster.ipp +++ b/thorlcr/graph/thgraphmaster.ipp @@ -204,9 +204,11 @@ public: CGraphTableCopy executed; CriticalSection exceptCrit; - virtual __int64 getWorkUnitValueInt(const char *prop, __int64 defVal) const; - virtual StringBuffer &getWorkUnitValue(const char *prop, StringBuffer &str) const; - virtual bool getWorkUnitValueBool(const char *prop, bool defVal) const; + virtual __int64 getWorkUnitValueInt(const char *prop, __int64 defVal) const override; + virtual StringBuffer &getWorkUnitValue(const char *prop, StringBuffer &str) const override; + virtual bool getWorkUnitValueBool(const char *prop, bool defVal) const override; + virtual double getWorkUnitValueReal(const char *prop, double defVal) const override; + // IExceptionHandler virtual bool fireException(IException *e); diff --git a/thorlcr/graph/thgraphslave.cpp b/thorlcr/graph/thgraphslave.cpp index 8e01d18d024..5f29b2c5c3b 100644 --- a/thorlcr/graph/thgraphslave.cpp +++ b/thorlcr/graph/thgraphslave.cpp @@ -1837,6 +1837,12 @@ bool CJobSlave::getWorkUnitValueBool(const char *prop, bool defVal) const return workUnitInfo->queryPropTree("Debug")->getPropBool(propName.toLowerCase().str(), defVal); } +double CJobSlave::getWorkUnitValueReal(const char *prop, double defVal) const +{ + StringBuffer propName(prop); + return workUnitInfo->queryPropTree("Debug")->getPropReal(propName.toLowerCase().str(), defVal); +} + void CJobSlave::debugRequest(MemoryBuffer &msg, const char *request) const { if (watchdog) watchdog->debugRequest(msg, request); diff --git a/thorlcr/graph/thgraphslave.hpp b/thorlcr/graph/thgraphslave.hpp index 9f6d557f975..b21a75dd96d 100644 --- a/thorlcr/graph/thgraphslave.hpp +++ b/thorlcr/graph/thgraphslave.hpp @@ -578,11 +578,12 @@ class graphslave_decl CJobSlave : public CJobBase void reportGraphEnd(graph_id gid); virtual mptag_t deserializeMPTag(MemoryBuffer &mb); - virtual __int64 getWorkUnitValueInt(const char *prop, __int64 defVal) const; - virtual StringBuffer &getWorkUnitValue(const char *prop, StringBuffer &str) const; - virtual bool getWorkUnitValueBool(const char *prop, bool defVal) const; - virtual IThorAllocator *getThorAllocator(unsigned channel); - virtual void debugRequest(MemoryBuffer &msg, const char *request) const; + virtual __int64 getWorkUnitValueInt(const char *prop, __int64 defVal) const override; + virtual StringBuffer &getWorkUnitValue(const char *prop, StringBuffer &str) const override; + virtual bool getWorkUnitValueBool(const char *prop, bool defVal) const override; + virtual double getWorkUnitValueReal(const char *prop, double defVal) const override; + virtual IThorAllocator *getThorAllocator(unsigned channel) override; + virtual void debugRequest(MemoryBuffer &msg, const char *request) const override; // IExceptionHandler virtual bool fireException(IException *e) diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index 1885c7029e4..3be0b1d230f 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -1686,6 +1686,13 @@ __int64 getExpertOptInt64(const char *opt, __int64 dft) return globals->getPropInt64(xpath, dft); } +double getExpertOptReal(const char *opt, double dft) +{ + StringBuffer xpath; + getExpertOptPath(opt, xpath); + return globals->getPropReal(xpath, dft); +} + StringBuffer &getExpertOptString(const char *opt, StringBuffer &out) { StringBuffer xpath; diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 5390d900892..6ce4466f8f6 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -607,6 +607,7 @@ extern graph_decl bool hasExpertOpt(const char *opt); extern graph_decl StringBuffer &getExpertOptPath(const char *opt, StringBuffer &out); extern graph_decl bool getExpertOptBool(const char *opt, bool dft=false); extern graph_decl __int64 getExpertOptInt64(const char *opt, __int64 dft=0); +extern graph_decl double getExpertOptReal(const char *opt, double dft); extern graph_decl StringBuffer &getExpertOptString(const char *opt, StringBuffer &out); extern graph_decl void setExpertOpt(const char *opt, const char *value); From eff15ffa5e5d41912a7e2eb7e30e031838772aac Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Mon, 7 Aug 2023 17:16:03 +0100 Subject: [PATCH 17/23] HPCC-30045 Move flame chart start/stop and allow interval to be set Signed-off-by: Jake Smith --- thorlcr/graph/thgraph.cpp | 32 --------------------------- thorlcr/graph/thgraphmaster.cpp | 10 +++++++++ thorlcr/slave/slavmain.cpp | 16 ++++++++++++++ thorlcr/thorutil/thormisc.cpp | 38 +++++++++++++++++++++++++++++++++ thorlcr/thorutil/thormisc.hpp | 10 +++++++++ 5 files changed, 74 insertions(+), 32 deletions(-) diff --git a/thorlcr/graph/thgraph.cpp b/thorlcr/graph/thgraph.cpp index 359d432586e..885c2ec0c74 100644 --- a/thorlcr/graph/thgraph.cpp +++ b/thorlcr/graph/thgraph.cpp @@ -1403,17 +1403,11 @@ void CGraphBase::executeSubGraph(size32_t parentExtractSz, const byte *parentExt CriticalBlock b(executeCrit); if (job.queryPausing()) return; - PerfTracer perf; Owned exception; try { if (!queryOwner()) { - if (job.getOptBool("perftrace")) - { - GraphPrintLog("Starting perf trace"); - perf.start(); - } if (!REJECTLOG(MCthorDetailedDebugInfo)) { StringBuffer s; @@ -1439,32 +1433,6 @@ void CGraphBase::executeSubGraph(size32_t parentExtractSz, const byte *parentExt getSystemTraceInfo(memStr, PerfMonStandard | PerfMonExtended); ::GraphPrintLog(this, thorDetailedLogLevel, "%s", memStr.str()); } - if (job.getOptBool("perftrace")) - { - GraphPrintLog("Stopping perf trace"); - perf.stop(); - StringBuffer flameGraphName; - if (getConfigurationDirectory(globals->queryPropTree("Directories"),"debug","thor",globals->queryProp("@name"),flameGraphName)) - addPathSepChar(flameGraphName); - flameGraphName.appendf("%s/%u/flame_%u.svg", job.queryWuid(), globals->getPropInt("@slavenum"), queryGraphId()); - ensureDirectoryForFile(flameGraphName); - Owned iFile = createIFile(flameGraphName); - try - { - Owned iFileIO = iFile->open(IFOcreate); - if (iFileIO) - { - StringBuffer &svg = perf.queryResult(); - iFileIO->write(0, svg.length(), svg.str()); - GraphPrintLog("Flame graph written to %s", flameGraphName.str()); - } - } - catch (IException *E) - { - GraphPrintLog(E); - ::Release(E); - } - } } if (exception) throw exception.getClear(); diff --git a/thorlcr/graph/thgraphmaster.cpp b/thorlcr/graph/thgraphmaster.cpp index b2e78c0d995..5874bfef515 100644 --- a/thorlcr/graph/thgraphmaster.cpp +++ b/thorlcr/graph/thgraphmaster.cpp @@ -2472,9 +2472,19 @@ void CMasterGraph::execute(size32_t _parentExtractSz, const byte *parentExtract, { if (isComplete()) return; + CThorPerfTracer perf; + double perfinterval = 0.0; if (!queryOwner()) // owning graph sends query+child graphs + { + perfinterval = job.getOptReal("perfInterval"); + if (perfinterval) + perf.start(job.queryWuid(), graphId, perfinterval); + jobM->sendQuery(); // if not previously sent + } CGraphBase::execute(parentExtractSz, parentExtract, checkDependencies, async); + if (perfinterval) + perf.stop(); } void CMasterGraph::start() diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index e1be240e844..4f99d2a06e2 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -1766,6 +1766,8 @@ class CJobListener : public CSimpleInterface CMessageBuffer msg; stopped = false; bool doReply; + + OwnedPtr perf; while (!stopped && queryNodeComm().recv(msg, 0, masterSlaveMpTag)) { doReply = true; @@ -1967,6 +1969,13 @@ class CJobListener : public CSimpleInterface msg.read(subGraphId); unsigned graphInitDataPos = msg.getPos(); + double perfInterval = job->getOptReal("perfInterval"); + if (perfInterval) + { + perf.setown(new CThorPerfTracer); + perf->start(job->queryWuid(), subGraphId, perfInterval); + } + VStringBuffer xpath("node[@id='%" GIDPF "u']", subGraphId); Owned graphNode = job->queryGraphXGMML()->getPropTree(xpath.str()); job->addSubGraph(*graphNode); @@ -2011,6 +2020,13 @@ class CJobListener : public CSimpleInterface { graph_id gid; msg.read(gid); + + if (perf) + { + perf->stop(); + perf.clear(); + } + msg.clear(); msg.append(false); for (unsigned c=0; cqueryJobChannels(); c++) diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index 3be0b1d230f..d0f53fda183 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -1710,3 +1710,41 @@ void setExpertOpt(const char *opt, const char *value) getExpertOptPath(opt, xpath.clear()); globals->setProp(xpath, value); } + + +void CThorPerfTracer::start(const char *_workunit, unsigned _subGraphId, double interval) +{ + workunit.set(_workunit); + subGraphId = _subGraphId; + PROGLOG("Starting perf trace of subgraph %u, with interval %.3g seconds", subGraphId, interval); + perf.setInterval(interval); + perf.start(); +} + + +void CThorPerfTracer::stop() +{ + PROGLOG("Stopping perf trace of subgraph %u", subGraphId); + perf.stop(); + StringBuffer flameGraphName; + if (getConfigurationDirectory(globals->queryPropTree("Directories"), "debug", "thor", globals->queryProp("@name"), flameGraphName)) + addPathSepChar(flameGraphName); + flameGraphName.appendf("%s/%u/flame_%u.svg", workunit.get(), globals->getPropInt("@slavenum"), subGraphId); + ensureDirectoryForFile(flameGraphName); + Owned iFile = createIFile(flameGraphName); + try + { + Owned iFileIO = iFile->open(IFOcreate); + if (iFileIO) + { + StringBuffer &svg = perf.queryResult(); + iFileIO->write(0, svg.length(), svg.str()); + PROGLOG("Flame graph for subgraph %u written to %s", subGraphId, flameGraphName.str()); + } + } + catch (IException *E) + { + EXCLOG(E); + ::Release(E); + } +} diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 6ce4466f8f6..2bfbc0d5998 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -713,5 +713,15 @@ class CThorContextLogger : public CSimpleInterfaceOf } }; +class graph_decl CThorPerfTracer : protected PerfTracer +{ + PerfTracer perf; + StringAttr workunit; + unsigned subGraphId; +public: + void start(const char *workunit, unsigned subGraphId, double interval); + void stop(); +}; + #endif From 3b36a5ee953485ff5d7691098dca2cf0e67d556d Mon Sep 17 00:00:00 2001 From: g-pan Date: Mon, 7 Aug 2023 15:25:19 -0400 Subject: [PATCH 18/23] HPCC-30046 Minor Changes to Containerized Doc Signed-off-by: g-pan --- .../ContainerizedHPCCSystemsPlatform.xml | 6 ++-- .../ContainerizedMods/AzureDeployment.xml | 31 ++++++++++--------- .../ContainerizedMods/ConfigureValues.xml | 13 ++++---- .../ContainerizedMods/ContainerLogging.xml | 4 +-- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml index 80a0c09b727..d1efc80c613 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedHPCCSystemsPlatform.xml @@ -130,7 +130,7 @@ few fundamental changes to note. - Processes and pods, not machines + Processes and Pods, not Machines Anyone familiar with the existing configuration system will know that part of the configuration involves creating instances of each @@ -149,7 +149,7 @@ - Helm charts + Helm Charts In the containerized world, the information that the operator needs to supply to configure an HPCC Systems environment is greatly @@ -204,7 +204,7 @@ - Topology settings – Clusters vs queues + Topology Settings – Clusters vs Queues In bare-metal deployments, there is a section called Topology where the various queues that diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/AzureDeployment.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/AzureDeployment.xml index 25cb4b9eb31..d79c28452a5 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/AzureDeployment.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/AzureDeployment.xml @@ -113,7 +113,7 @@ clusters in one resource group can cause confusion and difficulties to manage. Unless you or someone in your organization has already created a resource group and specified to work in that pre-existing resource - group, you will need to create one. + group, you will need to create one. To create a new resource group, you must choose a name and an Azure location. Additionally, you may choose to use tags for ease of @@ -202,7 +202,7 @@ only for the core AKS services and workloads. You will need to use a node taint on the newly created system node pool. Since you can't add taints to any pre-existing node pool, swap the default system node - pool for the newly created one. + pool for the newly created one. In order to do this, enter the following command (all on one line, if possible, and remove the connectors "\" as they are only @@ -269,7 +269,7 @@ - Installing the Helm charts + Installing the Helm Charts This section will demonstrate how to fetch, modify, and deploy the HPCC Systems charts. First we will need to access the HPCC Systems @@ -288,7 +288,7 @@ allows you to get the latest versions of the chart dependencies. - Installing the HPCC Systems components + Installing the HPCC Systems Components In order for a even a basic installation to succeed, it must have some type of storage enabled. The following steps will create @@ -296,7 +296,7 @@ that will allow the HPCC Systems to start and run but will not persist. To do this we will deploy the hpcc-azurefile chart which will set up Azure's ephemeral storage for the - HPCC Systems deployment. + HPCC Systems deployment. To Install the hpcc-azurefile chart: @@ -353,7 +353,7 @@ sasha: The indentation, syntax, and characters are very critical, please be sure those are an exact match to the above sample. A single extra space in this file can cause - unnecessary headaches. + unnecessary headaches. We can now use this @@ -397,7 +397,7 @@ sasha: key-value pair are very critical. Please be sure these are an exact match to the sample below. A single extra space, or missing character in this file can cause unnecessary - headaches. + headaches. @@ -516,9 +516,12 @@ externally, while eclservices is designed for internal use. In order to access the HPCC Systems services you must override these default settings to make them visible. We will now set the - visibility for eclwatch and eclqueries from local to global as in the - below example. Edit the myesp.yaml file and - change the two sections highlighted in the code examples below: + visibility for eclwatch and + eclqueries from local to global as in the below + example. + + Edit the myesp.yaml file and change the two + sections highlighted in the code examples below: esp: - name: eclwatch @@ -536,7 +539,7 @@ externally, while eclservices is designed for internal use. visibility: global ## Annotations can be specified on a service - for example to specify provider-specific i - + - name: eclqueries application: eclqueries @@ -554,7 +557,7 @@ externally, while eclservices is designed for internal use. - Install the customized HPCC Systems chart + Install the Customized HPCC Systems Chart This section will install the delivered HPCC Systems chart where we supply the myesp.yaml and @@ -654,7 +657,7 @@ mydali ClusterIP 10.0.195.229 <none> 7070/TCP 11m losing your data. The various storage options and strategies are discussed elsewhere - in addition to the HPCC Systems documentation. + in addition to the HPCC Systems documentation. Stopping Your HPCC Systems Cluster @@ -680,7 +683,7 @@ mydali ClusterIP 10.0.195.229 <none> 7070/TCP 11m helm uninstall myhpcccluster This will remove the HPCC Systems cluster named - <myhpcccluster> you had previously deployed. + <myhpcccluster> you had previously deployed. diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml index dadb6b59b4b..a8d5d801290 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml @@ -164,7 +164,7 @@ a manner that corresponds best to your operational strategy. - The Systems services + The Systems Services Most of the HPCC Systems components have a service definition entry, similar to the resources entry. All the components that have @@ -798,8 +798,7 @@ You can configure your HPCC Systems cloud deployment to access logical files from other remote environments. You configure this - remote storage by adding a "remote" section to your helm chart. - + remote storage by adding a "remote" section to your helm chart. The storage.remote section is a list of named remote environments that define the remote service url and a @@ -1237,7 +1236,7 @@ thor: The following example sets the environment variable to skip - Python cleanup on the Thor component: + Python cleanup on the Thor component: thor: env: @@ -1300,7 +1299,7 @@ thor: containerized version of the HPCC Systems platform. - The <emphasis>values.yaml</emphasis> file structure + The <emphasis>values.yaml</emphasis> File Structure The values.yaml file is a YAML file which is a format frequently used for configuration files. The construct that @@ -1416,7 +1415,7 @@ global: - HPCC Systems Values.yaml File Usage + HPCC Systems values.yaml File Usage The HPCC Systems values.yaml file is used by the Helm chart to control how HPCC Systems is deployed. The stock @@ -1462,7 +1461,7 @@ global: any properties defined for bundles. - Generally applicable + Generally Applicable These items are generally applicable for our HPCC Systems Helm YAML files. diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml index 404ed59a434..5cbef70cf29 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml @@ -485,7 +485,7 @@ global: - Provide AAD registered application information + Provide AAD Registered Application Information HPCC Systems logAccess requires access to the AAD tenant, client, token, and target workspace ID via secure secret object. The @@ -688,7 +688,7 @@ ContainerLog - Asychronous logging configuration + Asychronous Logging Configuration By default log entries will be created and logged asynchronously, so as not to block the client that is logging. Log entries will be held From 6fe0b49f8eb3ec126e0d80b5df0286f95d6714d7 Mon Sep 17 00:00:00 2001 From: Anthony Fishbeck Date: Mon, 7 Aug 2023 19:42:46 -0400 Subject: [PATCH 19/23] HPCC-30047 Vault class sets wrong authtype in token (client-secret) mode Signed-off-by: Anthony Fishbeck --- system/jlib/jsecrets.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index 6819a71ba45..766ef4d8b76 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -350,7 +350,7 @@ class CVault StringBuffer tokenText; if (getSecretKeyValue(clientToken, clientSecret, "token")) { - authType = VaultAuthType::appRole; + authType = VaultAuthType::token; PROGLOG("using a client token for vault auth"); } } From f8374cae3fc7ab1b3d65a95bd148a832c3ee32bb Mon Sep 17 00:00:00 2001 From: Kunal Aswani Date: Tue, 8 Aug 2023 12:56:34 +0200 Subject: [PATCH 20/23] HPCC-21665 Event Scheduler filter option added. Mine filter added to event scheduler page. Signed-off-by: Kunal Aswani --- esp/src/src-react/components/EventScheduler.tsx | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/esp/src/src-react/components/EventScheduler.tsx b/esp/src/src-react/components/EventScheduler.tsx index c3196a7706a..edbe34384d3 100644 --- a/esp/src/src-react/components/EventScheduler.tsx +++ b/esp/src/src-react/components/EventScheduler.tsx @@ -14,6 +14,7 @@ import { Filter } from "./forms/Filter"; import { PushEventForm } from "./forms/PushEvent"; import { ShortVerticalDivider } from "./Common"; import { QuerySortItem } from "src/store/Store"; +import { useMyAccount } from "../hooks/user"; const logger = scopedLogger("src-react/components/EventScheduler.tsx"); @@ -52,6 +53,7 @@ export const EventScheduler: React.FunctionComponent = ({ const [showFilter, setShowFilter] = React.useState(false); const [showPushEvent, setShowPushEvent] = React.useState(false); + const { currentUser } = useMyAccount(); // Grid --- const query = React.useMemo(() => { @@ -138,7 +140,19 @@ export const EventScheduler: React.FunctionComponent = ({ key: "pushEvent", text: nlsHPCC.PushEvent, onClick: () => setShowPushEvent(true) }, - ], [hasFilter, refreshTable, selection, setShowDescheduleConfirm, store]); + { key: "divider_3", itemType: ContextualMenuItemType.Divider, onRender: () => }, + { + key: "mine", text: nlsHPCC.Mine, disabled: !currentUser?.username, iconProps: { iconName: "Contact" }, canCheck: true, checked: filter["Owner"] === currentUser.username, + onClick: () => { + if (filter["Owner"] === currentUser.username) { + filter["Owner"] = ""; + } else { + filter["Owner"] = currentUser.username; + } + pushParams(filter); + } + }, + ], [currentUser, filter, hasFilter, refreshTable, selection, setShowDescheduleConfirm, store]); return } From d28ed29b5c4464da612c669db253d34b1b6c5570 Mon Sep 17 00:00:00 2001 From: Kunal Aswani Date: Wed, 9 Aug 2023 10:24:27 +0200 Subject: [PATCH 21/23] HPCC-19213 ECL Watch: Workunit list filter for 'protected' Added "Protected" fiter option to workunits page. Signed-off-by: Kunal Aswani --- esp/src/src-react/components/Workunits.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index b5939c074e7..09d104ac280 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -21,6 +21,7 @@ const logger = scopedLogger("src-react/components/Workunits.tsx"); const FilterFields: Fields = { "Type": { type: "checkbox", label: nlsHPCC.ArchivedOnly }, + "Protected": { type: "checkbox", label: nlsHPCC.Protected}, "Wuid": { type: "string", label: nlsHPCC.WUID, placeholder: "W20200824-060035" }, "Owner": { type: "string", label: nlsHPCC.Owner, placeholder: nlsHPCC.jsmi }, "Jobname": { type: "string", label: nlsHPCC.JobName, placeholder: nlsHPCC.log_analysis_1 }, @@ -57,6 +58,9 @@ function formatQuery(_filter): { [id: string]: any } { if (filter.Type === true) { filter.Type = "archived workunits"; } + if (filter.Protected === true) { + filter.Protected = "Protected"; + } logger.debug(filter); return filter; } From f8aca6601aace994685867d6879e7d2f1a2e91ed Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 11 Jul 2023 19:20:59 +0100 Subject: [PATCH 22/23] HPCC-29932 Avoid unusual crash for IF(cond,stmt) and cond is false Signed-off-by: Gavin Halliday --- ecl/hqlcpp/hqlcpp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecl/hqlcpp/hqlcpp.cpp b/ecl/hqlcpp/hqlcpp.cpp index ae23c63afb8..41fa0787524 100644 --- a/ecl/hqlcpp/hqlcpp.cpp +++ b/ecl/hqlcpp/hqlcpp.cpp @@ -7923,7 +7923,7 @@ void HqlCppTranslator::doBuildStmtIf(BuildCtx & ctx, IHqlExpression * expr) { if (matchesConstValue(boundCond->queryExpr(), true)) buildStmt(ctx, trueExpr); - else + else if (falseExpr) buildStmt(ctx, falseExpr); return; } From e52bc12c6989bda0da164b193cc6e9e71de3b64f Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:19:06 -0400 Subject: [PATCH 23/23] HPCC-30068 ECL Watch fix account_type not sent in request when changing permission on "Active Permissions" pages Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src/ws_access.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/esp/src/src/ws_access.ts b/esp/src/src/ws_access.ts index 2f4ea8903d3..d10a994431c 100644 --- a/esp/src/src/ws_access.ts +++ b/esp/src/src/ws_access.ts @@ -266,6 +266,7 @@ class AccountResourcesStore extends Memory { BasednName: row.BasednName, rname: row.ResourceName, account_name: row.account_name, + account_type: this.IsGroup ? 1 : 0, action: "update" }; lang.mixin(request, row);