Merge remote-tracking branch 'origin/candidate-9.2.x'

Signed-off-by: Jake Smith <[email protected]>
hpcc-systems · Jul 6, 2023 · 908a215 · 908a215
2 parents f179fd5 + fb10622
commit 908a215
Show file tree

Hide file tree

Showing 47 changed files with 422 additions and 178 deletions.
diff --git a/common/thorhelper/thorcommon.hpp b/common/thorhelper/thorcommon.hpp
@@ -278,10 +278,10 @@ class ActivityTimer
     ActivityTimer(ActivityTimeAccumulator &_accumulator, const bool _enabled)
     : accumulator(_accumulator), enabled(_enabled), isFirstRow(false)
     {
-        if (enabled)
+        if (likely(enabled))
         {
             startCycles = get_cycles_now();
-            if (!accumulator.firstRow)
+            if (unlikely(!accumulator.firstRow))
             {
                 isFirstRow = true;
                 accumulator.startCycles = startCycles;
@@ -294,13 +294,13 @@ class ActivityTimer
 
     ~ActivityTimer()
     {
-        if (enabled)
+        if (likely(enabled))
         {
             cycle_t nowCycles = get_cycles_now();
             accumulator.endCycles = nowCycles;
             cycle_t elapsedCycles = nowCycles - startCycles;
             accumulator.totalCycles += elapsedCycles;
-            if (isFirstRow)
+            if (unlikely(isFirstRow))
                 accumulator.firstExitCycles = nowCycles;
         }
     }
@@ -316,15 +316,15 @@ class SimpleActivityTimer
     inline SimpleActivityTimer(cycle_t &_accumulator, const bool _enabled)
     : accumulator(_accumulator), enabled(_enabled)
     {
-        if (enabled)
+        if (likely(enabled))
             startCycles = get_cycles_now();
         else
             startCycles = 0;
     }
 
     inline ~SimpleActivityTimer()
     {
-        if (enabled)
+        if (likely(enabled))
         {
             cycle_t nowCycles = get_cycles_now();
             cycle_t elapsedCycles = nowCycles - startCycles;

diff --git a/common/thorhelper/thorfile.cpp b/common/thorhelper/thorfile.cpp
@@ -242,6 +242,8 @@ static void gatherDerivedIndexInformation(DerivedIndexInformation & result, IDis
         result.sizeDiskLeaves = result.numLeafNodes * nodeSize;
         result.sizeDiskBlobs = result.numBlobNodes * nodeSize;
         result.sizeDiskBranches = result.numBranchNodes * nodeSize;
+        result.sizeMemoryBranches = attrs.getPropInt64("@branchMemorySize");
+        result.sizeMemoryLeaves = attrs.getPropInt64("@leafMemorySize");
     }
     else
     {
@@ -283,11 +285,12 @@ static void gatherDerivedIndexInformation(DerivedIndexInformation & result, IDis
         result.sizeOriginalData = attrs.getPropInt64("@uncompressedSize");
 
     //The following will depend on the compression format - e.g. if compressed searching is implemented
-    result.sizeMemoryBranches = result.sizeOriginalBranches;
+    if (result.sizeMemoryBranches == 0)
+        result.sizeMemoryBranches = result.sizeOriginalBranches;
 
     //NOTE: sizeOriginalData now includes the blob sizes that are removed before passing to the builder
     //      if the original blob size is recorded then use it, otherwise estimate it
-    if (result.sizeOriginalData)
+    if (result.sizeOriginalData && (result.sizeMemoryLeaves == 0))
     {
         offset_t originalBlobSize = attrs.getPropInt64("@originalBlobSize");
         if (result.numBlobNodes == 0)

diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp
@@ -2696,6 +2696,45 @@ cost_type aggregateDiskAccessCost(const IConstWorkUnit * wu, const char *scope)
     return totalCost;
 }
 
+void gatherSpillSize(const IConstWorkUnit * wu, const char *scope, stat_type & peakSizeSpill)
+{
+    WuScopeFilter filter;
+    if (!isEmptyString(scope))
+        filter.addScope(scope);
+    else
+    {
+        filter.addScope("");
+        filter.addSource("global");
+    }
+    filter.setIncludeNesting(1);
+    filter.addOutputStatistic(StSizeGraphSpill);
+    filter.addRequiredStat(StSizeGraphSpill);
+    filter.finishedFilter();
+    Owned<IConstWUScopeIterator> it = &wu->getScopeIterator(filter);
+    peakSizeSpill = 0;
+    for (it->first(); it->isValid(); )
+    {
+        stat_type value = 0;
+        if (it->getStat(StSizeGraphSpill, value))
+        {
+            if (value>peakSizeSpill)
+                peakSizeSpill = value;
+            it->nextSibling();
+        }
+        else
+        {
+            it->next();
+        }
+    }
+}
+
+void updateSpillSize(IWorkUnit * wu, const char * scope, StatisticScopeType scopeType)
+{
+    stat_type peakSizeSpill = 0;
+    gatherSpillSize(wu, scope, peakSizeSpill);
+    if (peakSizeSpill)
+        wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, StSizeGraphSpill, nullptr, peakSizeSpill, 1, 0, StatsMergeMax);
+}
 //---------------------------------------------------------------------------------------------------------------------
 
 

diff --git a/common/workunit/workunit.hpp b/common/workunit/workunit.hpp
@@ -1722,6 +1722,7 @@ extern WORKUNIT_API void updateWorkunitTimings(IWorkUnit * wu, StatisticScopeTyp
 extern WORKUNIT_API void aggregateStatistic(StatsAggregation & result, IConstWorkUnit * wu, const WuScopeFilter & filter, StatisticKind search);
 extern WORKUNIT_API cost_type aggregateCost(const IConstWorkUnit * wu, const char *scope=nullptr, bool excludehThor=false);
 extern WORKUNIT_API cost_type aggregateDiskAccessCost(const IConstWorkUnit * wu, const char *scope);
+extern WORKUNIT_API void updateSpillSize(IWorkUnit * wu, const char * scope, StatisticScopeType scopeType);
 extern WORKUNIT_API const char *getTargetClusterComponentName(const char *clustname, const char *processType, StringBuffer &name);
 extern WORKUNIT_API void descheduleWorkunit(char const * wuid);
 #if 0

diff --git a/ecl/eclagent/eclagent.cpp b/ecl/eclagent/eclagent.cpp
@@ -1985,7 +1985,7 @@ void EclAgent::doProcess()
         const cost_type diskAccessCost = aggregateDiskAccessCost(w, nullptr);
         if (diskAccessCost)
             w->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTglobal, "", StCostFileAccess, NULL, diskAccessCost, 1, 0, StatsMergeReplace);
-
+        updateSpillSize(w, nullptr, SSTglobal);
         addTimings(w);
 
         switch (w->getState())
@@ -2534,6 +2534,7 @@ void EclAgentWorkflowMachine::noteTiming(unsigned wfid, timestamp_type startTime
     const cost_type diskAccessCost = aggregateDiskAccessCost(wu, scope);
     if (diskAccessCost)
         wu->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), SSTworkflow, scope, StCostFileAccess, NULL, diskAccessCost, 1, 0, StatsMergeReplace);
+    updateSpillSize(wu, scope, SSTworkflow);
 }
 
 void EclAgentWorkflowMachine::doExecutePersistItem(IRuntimeWorkflowItem & item)

diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp
@@ -1161,6 +1161,8 @@ void CHThorIndexWriteActivity::execute()
     unsigned __int64 numBlobNodes = 0;
     unsigned __int64 numBranchNodes = 0;
     offset_t originalBlobSize = 0;
+    offset_t branchMemorySize = 0;
+    offset_t leafMemorySize = 0;
     unsigned nodeSize = 0;
 
     file.setown(createIFile(filename.get()));
@@ -1260,6 +1262,8 @@ void CHThorIndexWriteActivity::execute()
         numBranchNodes = builder->getNumBranchNodes();
         numBlobNodes = builder->getNumBlobNodes();
         originalBlobSize = bc.queryTotalSize();
+        branchMemorySize = builder->getBranchMemorySize();
+        leafMemorySize = builder->getLeafMemorySize();
 
         totalLeafNodes += numLeafNodes;
         totalBranchNodes += numBranchNodes;
@@ -1346,6 +1350,10 @@ void CHThorIndexWriteActivity::execute()
     properties.setPropInt64("@numBlobNodes", numBlobNodes);
     if (numBlobNodes)
         properties.setPropInt64("@originalBlobSize", originalBlobSize);
+    if (branchMemorySize)
+        properties.setPropInt64("@branchMemorySize", branchMemorySize);
+    if (leafMemorySize)
+        properties.setPropInt64("@leafMemorySize", leafMemorySize);
 
     size32_t keyedSize = helper.getKeyedSize();
     if (keyedSize == (size32_t)-1)

diff --git a/esp/services/ws_access/ws_accessService.cpp b/esp/services/ws_access/ws_accessService.cpp
@@ -1768,11 +1768,6 @@ bool Cws_accessEx::onResources(IEspContext &context, IEspResourcesRequest &req,
                     oneresource->setIsSpecial(true);
                 }
             }
-            else if(rtype == RT_FILE_SCOPE && stricmp(rname, "file") == 0)
-            {
-                //oneresource->setIsSpecial(true); //33067
-                continue;
-            }
 
             oneresource->setName(rname);
             oneresource->setDescription(r.getDescription());

diff --git a/esp/src/eclwatch/ClusterProcessesQueryWidget.js b/esp/src/eclwatch/ClusterProcessesQueryWidget.js
@@ -80,10 +80,14 @@ define([
         initTab: function () {
             var currSel = this.getSelectedChild();
             if (currSel && !currSel.initalized) {
-                if (currSel.id === this.legacyClustersProcessesIframeWidget.id && !this.legacyClustersProcessesIframeWidget.initalized) {
+                if (currSel.id === this.id + "_Grid") {
+                    this.refreshGrid();
+                } else if (currSel.id === this.legacyClustersProcessesIframeWidget.id && !this.legacyClustersProcessesIframeWidget.initalized) {
                     this.legacyClustersProcessesIframeWidget.init({
                         src: ESPRequest.getBaseURL("WsTopology") + "/TpClusterQuery?Type=ROOT"
                     });
+                } else {
+                    currSel.init(currSel.params);
                 }
             }
         },

diff --git a/esp/src/eclwatch/SystemServersQueryWidget.js b/esp/src/eclwatch/SystemServersQueryWidget.js
@@ -68,10 +68,14 @@ define([
         initTab: function () {
             var currSel = this.getSelectedChild();
             if (currSel && !currSel.initalized) {
-                if (currSel.id === this.systemServersQueryWidgetIframeWidget.id && !this.systemServersQueryWidgetIframeWidget.initalized) {
+                if (currSel.id === this.id + "_Grid") {
+                    this.refreshGrid();
+                } else if (currSel.id === this.systemServersQueryWidgetIframeWidget.id && !this.systemServersQueryWidgetIframeWidget.initalized) {
                     this.systemServersQueryWidgetIframeWidget.init({
                         src: ESPRequest.getBaseURL("WsTopology") + "/TpServiceQuery?Type=ALLSERVICES"
                     });
+                } else {
+                    currSel.init(currSel.params);
                 }
             }
         },
@@ -278,7 +282,7 @@ define([
             retVal.on(".dgrid-cell .gridClick:click", function (evt) {
                 var item = retVal.row(evt).data;
                 if (evt.target.title === "Audit Log" || evt.target.title === "Component Log") {
-                    context._onOpenLog(item);
+                    context._onOpenLog(item, evt.target.title);
                 } else {
                     context._onOpenConfiguration(item);
                 }
@@ -349,11 +353,11 @@ define([
             });
         },
 
-        _onOpenLog: function (item) {
+        _onOpenLog: function (item, type) {
             var nodeTab = this.ensureLogsPane(item.Name + ": " + item.Parent.LogDirectory, {
                 params: item,
                 ParentName: item.Parent.Name,
-                LogDirectory: item.Parent.LogDirectory,
+                LogDirectory: type === "Audit Log" ? item.Parent.AuditLogDirectory : item.Parent.LogDirectory,
                 NetAddress: item.Netaddress,
                 OS: item.OS,
                 newPreflight: true

diff --git a/esp/src/eclwatch/TargetClustersQueryWidget.js b/esp/src/eclwatch/TargetClustersQueryWidget.js
@@ -56,10 +56,14 @@ define([
         initTab: function () {
             var currSel = this.getSelectedChild();
             if (currSel && !currSel.initalized) {
-                if (currSel.id === this.legacyTargetClustersIframeWidget.id && !this.legacyTargetClustersIframeWidget.initalized) {
+                if (currSel.id === this.id + "_Grid") {
+                    this.refreshGrid();
+                } else if (currSel.id === this.legacyTargetClustersIframeWidget.id && !this.legacyTargetClustersIframeWidget.initalized) {
                     this.legacyTargetClustersIframeWidget.init({
                         src: ESPRequest.getBaseURL("WsTopology") + "/TpTargetClusterQuery?Type=ROOT"
                     });
+                } else if (currSel.params.newPreflight || currSel.params.Usergenerated) { //prevents loop of pfTab.init above
+                    currSel.init(currSel.params);
                 }
             }
         },

diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx
@@ -1,5 +1,6 @@
 import * as React from "react";
 import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Icon, Link } from "@fluentui/react";
+import { scopedLogger } from "@hpcc-js/util";
 import * as WsDfu from "src/WsDfu";
 import { CreateDFUQueryStore } from "src/ESPLogicalFile";
 import { formatCost } from "src/Session";
@@ -21,6 +22,8 @@ import { RenameFile } from "./forms/RenameFile";
 import { ShortVerticalDivider } from "./Common";
 import { SizeMe } from "react-sizeme";
 
+const logger = scopedLogger("src-react/components/Files.tsx");
+
 const FilterFields: Fields = {
     "LogicalName": { type: "string", label: nlsHPCC.Name, placeholder: nlsHPCC.somefile },
     "Description": { type: "string", label: nlsHPCC.Description, placeholder: nlsHPCC.SomeDescription },
@@ -218,7 +221,13 @@ export const Files: React.FunctionComponent<FilesProps> = ({
         message: nlsHPCC.DeleteSelectedFiles,
         items: selection.map(s => s.Name),
         onSubmit: React.useCallback(() => {
-            WsDfu.DFUArrayAction(selection, "Delete").then(() => refreshTable(true));
+            WsDfu.DFUArrayAction(selection, "Delete")
+                .then(({ DFUArrayActionResponse }) => {
+                    const ActionResults = DFUArrayActionResponse?.ActionResults?.DFUActionInfo ?? [];
+                    ActionResults.filter(action => action?.Failed).forEach(action => logger.error(action?.ActionResult));
+                    refreshTable(true);
+                })
+                .catch(err => logger.error(err));
         }, [refreshTable, selection])
     });
 

diff --git a/esp/src/src-react/components/LogicalFileSummary.tsx b/esp/src/src-react/components/LogicalFileSummary.tsx
@@ -59,6 +59,8 @@ export const LogicalFileSummary: React.FunctionComponent<LogicalFileSummaryProps
                 const actionInfo = ActionResults?.DFUActionInfo;
                 if (actionInfo && actionInfo.length && !actionInfo[0].Failed) {
                     replaceUrl("/files");
+                } else {
+                    logger.error(actionInfo[0].ActionResult);
                 }
             }).catch(err => logger.error(err));
         }, [file])

diff --git a/esp/src/src-react/components/SuperFileSummary.tsx b/esp/src/src-react/components/SuperFileSummary.tsx
@@ -46,7 +46,7 @@ export const SuperFileSummary: React.FunctionComponent<SuperFileSummaryProps> =
                 action: "remove",
                 superfile: file.Name,
                 subfiles: { Item: subfiles.map(file => file.Name) },
-                delete: true
+                removeSuperfile: true
             })
                 .then(() => replaceUrl("/files"))
                 .catch(err => logger.error(err))

diff --git a/esp/src/src/ESPLogicalFile.ts b/esp/src/src/ESPLogicalFile.ts
@@ -510,7 +510,7 @@ export function CreateDFUQueryStore(): BaseStore<HPCCWsDfu.DFUQueryRequest, type
         request.IncludeSuperOwner = true;
         return service.DFUQuery(request).then(response => {
             return {
-                data: response.DFULogicalFiles.DFULogicalFile,
+                data: response?.DFULogicalFiles?.DFULogicalFile ?? [],
                 total: response.NumFiles
             };
         });

diff --git a/helm/examples/efs/delete-role.sh b/helm/examples/efs/delete-role.sh
@@ -3,6 +3,7 @@ WORK_DIR=$(dirname $0)
 
 source ${WORK_DIR}/efs-env
 
-aws iam detach-role-policy --role-name ${EKS_NAME}_EFS_CSI_Role \
-  --policy-arn arn:aws:iam::${ACCOUNT_ID}:policy/AmazonEKS_EFS_CSI_Driver_Policy
-aws iam delete-role --role-name ${EKS_NAME}_EFS_CSI_Role
+echo "deleting iam role"
+echo "make sure you also uninstall the aws-efs-csi-driver helm chart"
+STACK_NAME=eksctl-${EKS_NAME}-addon-iamserviceaccount-kube-system-efs-csi-controller-sa
+aws cloudformation delete-stack --stack-name ${STACK_NAME}
diff --git a/helm/examples/efs/iam-policy-example.json b/helm/examples/efs/iam-policy-example.json
@@ -23,6 +23,18 @@
         }
       }
     },
+    {
+      "Effect": "Allow",
+      "Action": [
+          "elasticfilesystem:TagResource"
+      ],
+      "Resource": "*",
+      "Condition": {
+          "StringLike": {
+              "aws:ResourceTag/efs.csi.aws.com/cluster": "true"
+          }
+      }
+    },
     {
       "Effect": "Allow",
       "Action": "elasticfilesystem:DeleteAccessPoint",