Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HPCC-33334 Use cache stats to calculate all index read activities #19503

Draft
wants to merge 3 commits into
base: candidate-9.10.x
Choose a base branch
from
+208 −45
Draft

HPCC-33334 Use cache stats to calculate all index read activities #19503

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c139c79
HPCC-32901 Make super file cost and reads/write updates consistent
shamser Jan 23, 2025
bb1b26b
Changes following review
shamser Feb 4, 2025
1cdeb50
HPCC-33334 Use cache stats to calculate all index read activities
shamser Feb 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 143 additions & 1 deletion dali/base/dadfs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5069,7 +5069,63 @@ protected: friend class CDistributedFilePart;
atRestCost = calcFileAtRestCost(cluster, sizeGB, fileAgeDays);
}
if (attrs)
accessCost = getReadCost(*attrs, cluster) + getWriteCost(*attrs, cluster);
accessCost = ::getReadCost(*attrs, cluster) + ::getWriteCost(*attrs, cluster);
}

virtual bool getNumReads(stat_type &numReads) const override
{
const IPropertyTree *attrs = root->queryPropTree("Attr");
numReads = attrs ? attrs->getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads)) : 0;
return numReads > 0;
}

virtual bool getNumWrites(stat_type &numWrites) const override
{
const IPropertyTree *attrs = root->queryPropTree("Attr");
numWrites = attrs ? attrs->getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites)) : 0;
return numWrites > 0;
}

virtual bool getReadCost(cost_type &cost, bool calculateIfMissing=false) const override
{
IPropertyTree *attrs = root->queryPropTree("Attr");
if (attrs)
{
cost = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFreadCost), (unsigned __int64)-1);
if ((unsigned __int64)-1 != cost)
return true;
if (calculateIfMissing)
{
StringBuffer clusterName;
// getClusterName isn't const function, so need to cast this to non-const
const_cast<CDistributedFile *>(this)->getClusterName(0, clusterName);
cost = calcReadCost(*attrs, clusterName.str());
return true;
}
}
cost = 0;
return false;
}

virtual bool getWriteCost(cost_type &cost, bool calculateIfMissing=false) const override
{
const IPropertyTree *attrs = root->queryPropTree("Attr");
if (attrs)
{
cost = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), (unsigned __int64)-1);
if ((unsigned __int64)-1 != cost)
return true;
if (calculateIfMissing)
{
StringBuffer clusterName;
// getClusterName isn't const function, so need to cast this to non-const
const_cast<CDistributedFile *>(this)->getClusterName(0, clusterName);
cost = calcWriteCost(*attrs, clusterName.str());
return true;
}
}
cost = 0;
return false;
}
};

Expand Down Expand Up @@ -6336,6 +6392,78 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
return true;
}

virtual bool getNumReads(stat_type &numReads) const override
{
const IPropertyTree *attrs = root->queryPropTree("Attr");
if (attrs && attrs->hasProp(getDFUQResultFieldName(DFUQRFnumDiskReads)))
numReads = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads));
else
{
numReads = 0;
ForEachItemIn(i,subfiles)
{
stat_type c;
if (subfiles.item(i).getNumReads(c))
numReads += c;
}
}
return numReads > 0;
}

virtual bool getNumWrites(stat_type &numWrites) const override
{
const IPropertyTree *attrs = root->queryPropTree("Attr");
if (attrs && attrs->hasProp(getDFUQResultFieldName(DFUQRFnumDiskWrites)))
numWrites = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites));
else
{
numWrites = 0;
ForEachItemIn(i,subfiles)
{
stat_type c;
if (subfiles.item(i).getNumWrites(c))
numWrites += c;
}
}
return numWrites > 0;
}

virtual bool getReadCost(cost_type &cost, bool calculateIfMissing=false) const override
{
const IPropertyTree *attrs = root->queryPropTree("Attr");
if (attrs && attrs->hasProp(getDFUQResultFieldName(DFUQRFreadCost)))
cost = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFreadCost));
else
{
cost = 0;
ForEachItemIn(i,subfiles)
{
cost_type c;
if (subfiles.item(i).getReadCost(c,calculateIfMissing))
cost += c;
}
}
return cost > 0;
}

virtual bool getWriteCost(cost_type &cost, bool calculateIfMissing=false) const override
{
const IPropertyTree *attrs = root->queryPropTree("Attr");
if (attrs && attrs->hasProp(getDFUQResultFieldName(DFUQRFwriteCost)))
cost = attrs->getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost));
else
{
cost = 0;
ForEachItemIn(i,subfiles)
{
cost_type c;
if (!subfiles.item(i).getWriteCost(c,calculateIfMissing))
cost += c;
}
}
return cost > 0;
}

virtual IDistributedSuperFile *querySuperFile() override
{
return this;
Expand Down Expand Up @@ -6562,6 +6690,10 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
attrs.removeProp("@minSkew");
attrs.removeProp("@maxSkewPart");
attrs.removeProp("@minSkewPart");
attrs.removeProp(getDFUQResultFieldName(DFUQRFnumDiskReads));
attrs.removeProp(getDFUQResultFieldName(DFUQRFnumDiskWrites));
attrs.removeProp(getDFUQResultFieldName(DFUQRFreadCost));
attrs.removeProp(getDFUQResultFieldName(DFUQRFwriteCost));

__int64 fs = getFileSize(false,false);
if (fs!=-1)
Expand Down Expand Up @@ -6589,6 +6721,16 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
attrs.setPropBin("_record_layout", mb.length(), mb.bufferBase());
if (getRecordLayout(mb, "_rtlType"))
attrs.setPropBin("_rtlType", mb.length(), mb.bufferBase());
stat_type numReads, numWrites;
if (getNumReads(numReads))
attrs.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), numReads);
if (getNumWrites(numWrites))
attrs.setPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), numWrites);
cost_type readCost, writeCost;
if (getReadCost(readCost))
attrs.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), numReads);
if (getWriteCost(writeCost))
attrs.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), numWrites);
const char *kind = nullptr;
Owned<IDistributedFileIterator> subIter = getSubFileIterator(true);
ForEach(*subIter)
Expand Down
75 changes: 36 additions & 39 deletions dali/base/dadfs.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,10 @@ interface IDistributedFile: extends IInterface
virtual int getExpire(StringBuffer *expirationDate) = 0;
virtual void setExpire(int expireDays) = 0;
virtual void getCost(const char * cluster, cost_type & atRestCost, cost_type & accessCost) = 0;
virtual bool getNumReads(stat_type &numReads) const = 0;
virtual bool getNumWrites(stat_type &numWrites) const = 0;
virtual bool getReadCost(cost_type &cost, bool calculateIfMissing=false) const = 0;
virtual bool getWriteCost(cost_type &cost, bool calculateIfMissing=false) const = 0;
};


Expand Down Expand Up @@ -896,6 +900,20 @@ constexpr bool defaultNonPrivilegedUser = false;

extern da_decl void configurePreferredPlanes();

template<typename Source>
inline cost_type calcReadCost(const IPropertyTree & fileAttr, Source source)
{
// Calculate legacy read cost from numDiskReads
// (However, it is not possible to accurately calculate read cost for key
// files, as the reads may have been from page cache and not from disk.)
if (!isFileKey(fileAttr) && source)
{
stat_type numDiskReads = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0);
return calcFileAccessCost(source, 0, numDiskReads);
}
return 0;
}

// Get read cost from readCost field or calculate legacy read cost
// - migrateLegacyCost: if true, update readCost field with legacy read cost
template<typename Source>
Expand All @@ -905,19 +923,11 @@ inline cost_type getReadCost(IPropertyTree & fileAttr, Source source, bool migra
return fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFreadCost), 0);
else
{
// Calculate legacy read cost from numDiskReads
// (However, it is not possible to accurately calculate read cost for key
// files, as the reads may have been from page cache and not from disk.)
if (!isFileKey(fileAttr) && source)
{
stat_type numDiskReads = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0);
cost_type readCost = calcFileAccessCost(source, 0, numDiskReads);
if (migrateLegacyCost)
fileAttr.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), readCost);
return readCost;
}
cost_type readCost = calcReadCost(fileAttr, source);
if (migrateLegacyCost)
fileAttr.setPropInt64(getDFUQResultFieldName(DFUQRFreadCost), readCost);
return readCost;
}
return 0;
}

// Get read cost from readCost field or calculate legacy read cost
Expand All @@ -927,15 +937,16 @@ inline cost_type getReadCost(const IPropertyTree & fileAttr, Source source)
if (fileAttr.hasProp(getDFUQResultFieldName(DFUQRFreadCost)))
return fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFreadCost), 0);
else
return calcReadCost(fileAttr, source);
}

template<typename Source>
inline cost_type calcWriteCost(const IPropertyTree & fileAttr, Source source)
{
if (source)
{
// Calculate legacy read cost from numDiskReads
// (However, it is not possible to accurately calculate read cost for key
// files, as the reads may have been from page cache and not from disk.)
if (!isFileKey(fileAttr) && source)
{
stat_type numDiskReads = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskReads), 0);
return calcFileAccessCost(source, 0, numDiskReads);
}
stat_type numDiskWrites = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), 0);
return calcFileAccessCost(source, numDiskWrites, 0);
}
return 0;
}
Expand All @@ -949,17 +960,11 @@ inline cost_type getWriteCost(IPropertyTree & fileAttr, Source source, bool migr
return fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), 0);
else
{
// Calculate legacy write cost from numDiskWrites
if (source)
{
stat_type numDiskWrites = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), 0);
cost_type writeCost = calcFileAccessCost(source, numDiskWrites, 0);
if (migrateLegacyCost)
fileAttr.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), writeCost);
return writeCost;
}
cost_type writeCost = calcWriteCost(fileAttr, source);
if (migrateLegacyCost)
fileAttr.setPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), writeCost);
return writeCost;
}
return 0;
}

// Get write cost from writeCost field or calculate legacy write cost
Expand All @@ -969,15 +974,7 @@ inline cost_type getWriteCost(const IPropertyTree & fileAttr, Source source)
if (fileAttr.hasProp(getDFUQResultFieldName(DFUQRFwriteCost)))
return fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFwriteCost), 0);
else
{
// Calculate legacy write cost from numDiskWrites
if (source)
{
stat_type numDiskWrites = fileAttr.getPropInt64(getDFUQResultFieldName(DFUQRFnumDiskWrites), 0);
return calcFileAccessCost(source, numDiskWrites, 0);
}
}
return 0;
return calcWriteCost(fileAttr, source);
}

extern da_decl bool doesPhysicalMatchMeta(IPartDescriptor &partDesc, IFile &iFile, offset_t &expectedSize, offset_t &actualSize);
Expand Down
1 change: 1 addition & 0 deletions ecl/hthor/hthor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8545,6 +8545,7 @@ void CHThorDiskReadBaseActivity::closepart()
{
IDistributedSuperFile *super = dFile->querySuperFile();
dFile = &(super->querySubFile(subfile, true));
updateCostAndNumReads(super, curDiskReads);
}
}
updateCostAndNumReads(dFile, curDiskReads);
Expand Down
17 changes: 16 additions & 1 deletion esp/clients/ws_dfsclient/ws_dfsclient.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,22 @@ class CServiceDistributedFileBase : public CSimpleInterfaceOf<INTERFACE>
virtual bool getSkewInfo(unsigned &maxSkew, unsigned &minSkew, unsigned &maxSkewPart, unsigned &minSkewPart, bool calculateIfMissing) override { return legacyDFSFile->getSkewInfo(maxSkew, minSkew, maxSkewPart, minSkewPart, calculateIfMissing); }
virtual int getExpire(StringBuffer *expirationDate) override { return legacyDFSFile->getExpire(expirationDate); }
virtual void getCost(const char * cluster, cost_type & atRestCost, cost_type & accessCost) override { legacyDFSFile->getCost(cluster, atRestCost, accessCost); }

virtual bool getNumReads(stat_type &numReads) const override
{
return legacyDFSFile->getNumReads(numReads);
}
virtual bool getNumWrites(stat_type &numWrites) const override
{
return legacyDFSFile->getNumWrites(numWrites);
}
virtual bool getReadCost(cost_type &cost, bool calculateIfMissing) const override
{
return legacyDFSFile->getReadCost(cost, calculateIfMissing);
}
virtual bool getWriteCost(cost_type &cost, bool calculateIfMissing) const override
{
return legacyDFSFile->getWriteCost(cost, calculateIfMissing);
}
// setters (change file meta data)
virtual void setPreferredClusters(const char *clusters) override { legacyDFSFile->setPreferredClusters(clusters); }
virtual void setSingleClusterOnly() override { legacyDFSFile->setSingleClusterOnly(); }
Expand Down
16 changes: 12 additions & 4 deletions thorlcr/graph/thgraphmaster.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -670,13 +670,17 @@ cost_type CMasterActivity::calcFileReadCostStats(bool updateFileProps)
curReadCost = calcFileAccessCost(clusterName, 0, curDiskReads);

if (updateFileProps)
updateCostAndNumReads(file, curDiskReads);
updateCostAndNumReads(file, curDiskReads, curReadCost);
return curReadCost;
};
cost_type readCost = 0;
ThorActivityKind actKind = container.getKind();
bool isIndexActivity = (TAKindexread == actKind) || (TAKkeyedjoin == actKind) || (TAKindexnormalize == actKind)
|| (TAKindexaggregate == actKind) || (TAKindexcount == actKind) || (TAKindexgroupaggregate == actKind)
|| (TAKindexgroupexists == actKind) || (TAKindexgroupcount == actKind);

if (fileStats.size()>0)
{
ThorActivityKind activityKind = container.getKind();
unsigned fileIndex = 0;
diskAccessCost = 0;
for (unsigned i=0; i<readFiles.size(); i++)
Expand All @@ -685,20 +689,24 @@ cost_type CMasterActivity::calcFileReadCostStats(bool updateFileProps)
bool useJhtreeCache = false;
// Index uses jhtree caches, so use actual fetches to calculate cost
// To determine entry is an index file entry, use the test (i==0) because index file is always the first file
if ((TAKindexread == activityKind) || ((TAKkeyedjoin == activityKind) && (0 == i)))
if (isIndexActivity && (!(TAKkeyedjoin == actKind) || (0 == i)))
useJhtreeCache = true;
if (file)
{
IDistributedSuperFile *super = file->querySuperFile();
if (super)
{
unsigned numSubFiles = super->numSubFiles(true);
stat_type curDiskReads = 0;
for (unsigned i=0; i<numSubFiles; i++)
{
IDistributedFile &subFile = super->querySubFile(i, true);
readCost += updateReadCosts(useJhtreeCache, &subFile, *fileStats[fileIndex]);
curDiskReads += fileStats[fileIndex]->getStatisticSum(StNumDiskReads);
fileIndex++;
}
if (updateFileProps)
updateCostAndNumReads(super, curDiskReads, readCost);
}
else
{
Expand All @@ -712,7 +720,7 @@ cost_type CMasterActivity::calcFileReadCostStats(bool updateFileProps)
{
IDistributedFile *file = queryReadFile(0);
if (file)
readCost = updateReadCosts(true, file, statsCollection);
readCost = updateReadCosts(isIndexActivity, file, statsCollection);
}
return readCost;
}
Expand Down