diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 734db18b27b..fb52bb3f02a 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -5069,7 +5069,63 @@ protected: friend class CDistributedFilePart; atRestCost = calcFileAtRestCost(cluster, sizeGB, fileAgeDays); } if (attrs) - accessCost = getReadCost(*attrs, cluster) + getWriteCost(*attrs, cluster); + accessCost = ::getReadCost(*attrs, cluster) + ::getWriteCost(*attrs, cluster); + } + + virtual bool getNumReads(stat_type &numReads) const override + { + const IPropertyTree *attrs = root->queryPropTree("Attr"); + numReads = attrs ? attrs->getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads)) : 0; + return numReads > 0; + } + + virtual bool getNumWrites(stat_type &numWrites) const override + { + const IPropertyTree *attrs = root->queryPropTree("Attr"); + numWrites = attrs ? attrs->getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites)) : 0; + return numWrites > 0; + } + + virtual bool getReadCost(cost_type &cost, bool calculateIfMissing=false) const override + { + IPropertyTree *attrs = root->queryPropTree("Attr"); + if (attrs) + { + cost = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFreadCost), (unsigned __int64)-1); + if ((unsigned __int64)-1 != cost) + return true; + if (calculateIfMissing) + { + StringBuffer clusterName; + // getClusterName isn't const function, so need to cast this to non-const + const_cast(this)->getClusterName(0, clusterName); + cost = calcReadCost(*attrs, clusterName.str()); + return true; + } + } + cost = 0; + return false; + } + + virtual bool getWriteCost(cost_type &cost, bool calculateIfMissing=false) const override + { + const IPropertyTree *attrs = root->queryPropTree("Attr"); + if (attrs) + { + cost = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), (unsigned __int64)-1); + if ((unsigned __int64)-1 != cost) + return true; + if (calculateIfMissing) + { + StringBuffer clusterName; + // getClusterName isn't const function, so need to cast this to non-const + const_cast(this)->getClusterName(0, clusterName); + cost = calcWriteCost(*attrs, clusterName.str()); + return true; + } + } + cost = 0; + return false; } }; @@ -6336,6 +6392,78 @@ class CDistributedSuperFile: public CDistributedFileBase return true; } + virtual bool getNumReads(stat_type &numReads) const override + { + const IPropertyTree *attrs = root->queryPropTree("Attr"); + if (attrs && attrs->hasProp(getDFUQResultFieldName(DFUQRFnumDiskReads))) + numReads = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads)); + else + { + numReads = 0; + ForEachItemIn(i,subfiles) + { + stat_type c; + if (subfiles.item(i).getNumReads(c)) + numReads += c; + } + } + return numReads > 0; + } + + virtual bool getNumWrites(stat_type &numWrites) const override + { + const IPropertyTree *attrs = root->queryPropTree("Attr"); + if (attrs && attrs->hasProp(getDFUQResultFieldName(DFUQRFnumDiskWrites))) + numWrites = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites)); + else + { + numWrites = 0; + ForEachItemIn(i,subfiles) + { + stat_type c; + if (subfiles.item(i).getNumWrites(c)) + numWrites += c; + } + } + return numWrites > 0; + } + + virtual bool getReadCost(cost_type &cost, bool calculateIfMissing=false) const override + { + const IPropertyTree *attrs = root->queryPropTree("Attr"); + if (attrs && attrs->hasProp(getDFUQResultFieldName(DFUQRFreadCost))) + cost = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFreadCost)); + else + { + cost = 0; + ForEachItemIn(i,subfiles) + { + cost_type c; + if (subfiles.item(i).getReadCost(c,calculateIfMissing)) + cost += c; + } + } + return cost > 0; + } + + virtual bool getWriteCost(cost_type &cost, bool calculateIfMissing=false) const override + { + const IPropertyTree *attrs = root->queryPropTree("Attr"); + if (attrs && attrs->hasProp(getDFUQResultFieldName(DFUQRFwriteCost))) + cost = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost)); + else + { + cost = 0; + ForEachItemIn(i,subfiles) + { + cost_type c; + if (!subfiles.item(i).getWriteCost(c,calculateIfMissing)) + cost += c; + } + } + return cost > 0; + } + virtual IDistributedSuperFile *querySuperFile() override { return this; @@ -6562,6 +6690,10 @@ class CDistributedSuperFile: public CDistributedFileBase attrs.removeProp("@minSkew"); attrs.removeProp("@maxSkewPart"); attrs.removeProp("@minSkewPart"); + attrs.removeProp(getDFUQResultFieldName(DFUQRFnumDiskReads)); + attrs.removeProp(getDFUQResultFieldName(DFUQRFnumDiskWrites)); + attrs.removeProp(getDFUQResultFieldName(DFUQRFreadCost)); + attrs.removeProp(getDFUQResultFieldName(DFUQRFwriteCost)); __int64 fs = getFileSize(false,false); if (fs!=-1) @@ -6589,6 +6721,16 @@ class CDistributedSuperFile: public CDistributedFileBase attrs.setPropBin("_record_layout", mb.length(), mb.bufferBase()); if (getRecordLayout(mb, "_rtlType")) attrs.setPropBin("_rtlType", mb.length(), mb.bufferBase()); + stat_type numReads, numWrites; + if (getNumReads(numReads)) + attrs.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), numReads); + if (getNumWrites(numWrites)) + attrs.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), numWrites); + cost_type readCost, writeCost; + if (getReadCost(readCost)) + attrs.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), numReads); + if (getWriteCost(writeCost)) + attrs.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), numWrites); const char *kind = nullptr; Owned subIter = getSubFileIterator(true); ForEach(*subIter) diff --git a/dali/base/dadfs.hpp b/dali/base/dadfs.hpp index fb8453b291a..985b2fb0c7e 100644 --- a/dali/base/dadfs.hpp +++ b/dali/base/dadfs.hpp @@ -434,6 +434,10 @@ interface IDistributedFile: extends IInterface virtual int getExpire(StringBuffer *expirationDate) = 0; virtual void setExpire(int expireDays) = 0; virtual void getCost(const char * cluster, cost_type & atRestCost, cost_type & accessCost) = 0; + virtual bool getNumReads(stat_type &numReads) const = 0; + virtual bool getNumWrites(stat_type &numWrites) const = 0; + virtual bool getReadCost(cost_type &cost, bool calculateIfMissing=false) const = 0; + virtual bool getWriteCost(cost_type &cost, bool calculateIfMissing=false) const = 0; }; @@ -896,6 +900,20 @@ constexpr bool defaultNonPrivilegedUser = false; extern da_decl void configurePreferredPlanes(); +template +inline cost_type calcReadCost(const IPropertyTree & fileAttr, Source source) +{ + // Calculate legacy read cost from numDiskReads + // (However, it is not possible to accurately calculate read cost for key + // files, as the reads may have been from page cache and not from disk.) + if (!isFileKey(fileAttr) && source) + { + stat_type numDiskReads = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0); + return calcFileAccessCost(source, 0, numDiskReads); + } + return 0; +} + // Get read cost from readCost field or calculate legacy read cost // - migrateLegacyCost: if true, update readCost field with legacy read cost template @@ -905,19 +923,11 @@ inline cost_type getReadCost(IPropertyTree & fileAttr, Source source, bool migra return fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFreadCost), 0); else { - // Calculate legacy read cost from numDiskReads - // (However, it is not possible to accurately calculate read cost for key - // files, as the reads may have been from page cache and not from disk.) - if (!isFileKey(fileAttr) && source) - { - stat_type numDiskReads = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0); - cost_type readCost = calcFileAccessCost(source, 0, numDiskReads); - if (migrateLegacyCost) - fileAttr.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), readCost); - return readCost; - } + cost_type readCost = calcReadCost(fileAttr, source); + if (migrateLegacyCost) + fileAttr.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), readCost); + return readCost; } - return 0; } // Get read cost from readCost field or calculate legacy read cost @@ -927,15 +937,16 @@ inline cost_type getReadCost(const IPropertyTree & fileAttr, Source source) if (fileAttr.hasProp(getDFUQResultFieldName(DFUQRFreadCost))) return fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFreadCost), 0); else + return calcReadCost(fileAttr, source); +} + +template +inline cost_type calcWriteCost(const IPropertyTree & fileAttr, Source source) +{ + if (source) { - // Calculate legacy read cost from numDiskReads - // (However, it is not possible to accurately calculate read cost for key - // files, as the reads may have been from page cache and not from disk.) - if (!isFileKey(fileAttr) && source) - { - stat_type numDiskReads = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0); - return calcFileAccessCost(source, 0, numDiskReads); - } + stat_type numDiskWrites = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), 0); + return calcFileAccessCost(source, numDiskWrites, 0); } return 0; } @@ -949,17 +960,11 @@ inline cost_type getWriteCost(IPropertyTree & fileAttr, Source source, bool migr return fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), 0); else { - // Calculate legacy write cost from numDiskWrites - if (source) - { - stat_type numDiskWrites = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), 0); - cost_type writeCost = calcFileAccessCost(source, numDiskWrites, 0); - if (migrateLegacyCost) - fileAttr.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), writeCost); - return writeCost; - } + cost_type writeCost = calcWriteCost(fileAttr, source); + if (migrateLegacyCost) + fileAttr.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), writeCost); + return writeCost; } - return 0; } // Get write cost from writeCost field or calculate legacy write cost @@ -969,15 +974,7 @@ inline cost_type getWriteCost(const IPropertyTree & fileAttr, Source source) if (fileAttr.hasProp(getDFUQResultFieldName(DFUQRFwriteCost))) return fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), 0); else - { - // Calculate legacy write cost from numDiskWrites - if (source) - { - stat_type numDiskWrites = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), 0); - return calcFileAccessCost(source, numDiskWrites, 0); - } - } - return 0; + return calcWriteCost(fileAttr, source); } extern da_decl bool doesPhysicalMatchMeta(IPartDescriptor &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize); diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index 6dd02b640c7..30fb379f14f 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -8545,6 +8545,7 @@ void CHThorDiskReadBaseActivity::closepart() { IDistributedSuperFile *super = dFile->querySuperFile(); dFile = &(super->querySubFile(subfile, true)); + updateCostAndNumReads(super, curDiskReads); } } updateCostAndNumReads(dFile, curDiskReads); diff --git a/esp/clients/ws_dfsclient/ws_dfsclient.cpp b/esp/clients/ws_dfsclient/ws_dfsclient.cpp index 81a845665a1..ce7148fa91b 100644 --- a/esp/clients/ws_dfsclient/ws_dfsclient.cpp +++ b/esp/clients/ws_dfsclient/ws_dfsclient.cpp @@ -236,7 +236,22 @@ class CServiceDistributedFileBase : public CSimpleInterfaceOf virtual bool getSkewInfo(unsigned &maxSkew, unsigned &minSkew, unsigned &maxSkewPart, unsigned &minSkewPart, bool calculateIfMissing) override { return legacyDFSFile->getSkewInfo(maxSkew, minSkew, maxSkewPart, minSkewPart, calculateIfMissing); } virtual int getExpire(StringBuffer *expirationDate) override { return legacyDFSFile->getExpire(expirationDate); } virtual void getCost(const char * cluster, cost_type & atRestCost, cost_type & accessCost) override { legacyDFSFile->getCost(cluster, atRestCost, accessCost); } - + virtual bool getNumReads(stat_type &numReads) const override + { + return legacyDFSFile->getNumReads(numReads); + } + virtual bool getNumWrites(stat_type &numWrites) const override + { + return legacyDFSFile->getNumWrites(numWrites); + } + virtual bool getReadCost(cost_type &cost, bool calculateIfMissing) const override + { + return legacyDFSFile->getReadCost(cost, calculateIfMissing); + } + virtual bool getWriteCost(cost_type &cost, bool calculateIfMissing) const override + { + return legacyDFSFile->getWriteCost(cost, calculateIfMissing); + } // setters (change file meta data) virtual void setPreferredClusters(const char *clusters) override { legacyDFSFile->setPreferredClusters(clusters); } virtual void setSingleClusterOnly() override { legacyDFSFile->setSingleClusterOnly(); } diff --git a/thorlcr/graph/thgraphmaster.cpp b/thorlcr/graph/thgraphmaster.cpp index 9dd7350afaa..a4fa975f5d8 100644 --- a/thorlcr/graph/thgraphmaster.cpp +++ b/thorlcr/graph/thgraphmaster.cpp @@ -670,13 +670,17 @@ cost_type CMasterActivity::calcFileReadCostStats(bool updateFileProps) curReadCost = calcFileAccessCost(clusterName, 0, curDiskReads); if (updateFileProps) - updateCostAndNumReads(file, curDiskReads); + updateCostAndNumReads(file, curDiskReads, curReadCost); return curReadCost; }; cost_type readCost = 0; + ThorActivityKind actKind = container.getKind(); + bool isIndexActivity = (TAKindexread == actKind) || (TAKkeyedjoin == actKind) || (TAKindexnormalize == actKind) + || (TAKindexaggregate == actKind) || (TAKindexcount == actKind) || (TAKindexgroupaggregate == actKind) + || (TAKindexgroupexists == actKind) || (TAKindexgroupcount == actKind); + if (fileStats.size()>0) { - ThorActivityKind activityKind = container.getKind(); unsigned fileIndex = 0; diskAccessCost = 0; for (unsigned i=0; inumSubFiles(true); + stat_type curDiskReads = 0; for (unsigned i=0; iquerySubFile(i, true); readCost += updateReadCosts(useJhtreeCache, &subFile, *fileStats[fileIndex]); + curDiskReads += fileStats[fileIndex]->getStatisticSum(StNumDiskReads); fileIndex++; } + if (updateFileProps) + updateCostAndNumReads(super, curDiskReads, readCost); } else { @@ -712,7 +720,7 @@ cost_type CMasterActivity::calcFileReadCostStats(bool updateFileProps) { IDistributedFile *file = queryReadFile(0); if (file) - readCost = updateReadCosts(true, file, statsCollection); + readCost = updateReadCosts(isIndexActivity, file, statsCollection); } return readCost; }