From f477f508a4fc940b33534659ac97de90a0c65061 Mon Sep 17 00:00:00 2001 From: Centis Biks Date: Tue, 3 Sep 2019 16:57:48 -0700 Subject: [PATCH] Issue microsoft#117: Implement more robust latency information of thread groups. . Added TargetID to make it possible to connect a / to it's original / in multi-threaded test Profiles. . Added latency totals and grouping of TargeID's. --- CmdLineParser/CmdLineParser.cpp | 4 ++- Common/Histogram.h | 21 +++++++-------- IORequestGenerator/IORequestGenerator.cpp | 1 + XmlProfileParser/XmlProfileParser.cpp | 3 ++- XmlResultParser/XmlResultParser.cpp | 31 ++++++++++++++++++++--- 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/CmdLineParser/CmdLineParser.cpp b/CmdLineParser/CmdLineParser.cpp index efc6b87..5a8700b 100644 --- a/CmdLineParser/CmdLineParser.cpp +++ b/CmdLineParser/CmdLineParser.cpp @@ -570,14 +570,16 @@ bool CmdLineParser::_ReadParametersFromCmdLine(const int argc, const char *argv[ // create targets vector vTargets; int iFirstFile = -1; + int iTargetID = 0; for (int i = 1; i < argc; i++) { if (argv[i][0] != '-' && argv[i][0] != '/') { iFirstFile = i; - Target target; + Target target(iTargetID); target.SetPath(argv[i]); vTargets.push_back(target); + iTargetID++; } } diff --git a/Common/Histogram.h b/Common/Histogram.h index 7d0b7cd..f8f1ecc 100644 --- a/Common/Histogram.h +++ b/Common/Histogram.h @@ -41,6 +41,9 @@ using HistogramBucketList = std::vector; using HistogramBucketListPtr = std::shared_ptr; using ConstHistogramBucketListPtr = std::shared_ptr; +/**************************************************************************************************************************************************** + CBTODO: Histograms consumers should use a specific type, like PerfTimer, rather than float/double to minimze conversion errors. +****************************************************************************************************************************************************/ template class Histogram { @@ -122,12 +125,11 @@ class Histogram { T value = std::numeric_limits::max(); - for (auto i : _data) + auto sortedData = _GetSortedData(); + auto iter = sortedData->begin(); + if (iter != sortedData->end()) { - if (i.first < value) - { - value = i.first; - } + value = iter->first; } return value; @@ -137,12 +139,11 @@ class Histogram { T value = std::numeric_limits::min(); - for (auto i : _data) + auto sortedData = _GetSortedData(); + auto iter = sortedData->rbegin(); + if (iter != sortedData->rend()) { - if (i.first > value) - { - value = i.first; - } + value = iter->first; } return value; diff --git a/IORequestGenerator/IORequestGenerator.cpp b/IORequestGenerator/IORequestGenerator.cpp index 52f22d6..626bc78 100644 --- a/IORequestGenerator/IORequestGenerator.cpp +++ b/IORequestGenerator/IORequestGenerator.cpp @@ -1552,6 +1552,7 @@ DWORD WINAPI threadFunc(LPVOID cookie) p->pResults->vTargetResults.resize(p->vTargets.size()); for (size_t i = 0; i < p->vullFileSizes.size(); i++) { + p->pResults->vTargetResults[i].iTargetID = p->vTargets[i].GetTargetID(); p->pResults->vTargetResults[i].sPath = p->vTargets[i].GetPath(); p->pResults->vTargetResults[i].ullFileSize = p->vullFileSizes[i]; if(fCalculateIopsStdDev) diff --git a/XmlProfileParser/XmlProfileParser.cpp b/XmlProfileParser/XmlProfileParser.cpp index 113ad0e..07df30e 100644 --- a/XmlProfileParser/XmlProfileParser.cpp +++ b/XmlProfileParser/XmlProfileParser.cpp @@ -601,7 +601,7 @@ HRESULT XmlProfileParser::_ParseTargets(IXMLDOMNode *pXmlNode, TimeSpan *pTimeSp hr = spNodeList->get_item(i, &spNode); if (SUCCEEDED(hr)) { - Target target; + Target target(i); _ParseTarget(spNode, &target); pTimeSpan->AddTarget(target); } @@ -1294,6 +1294,7 @@ HRESULT XmlProfileParser::_GetProgress(IXMLDOMDocument2 *pXmlDoc, DWORD *pdwProg { return _GetDWORD(pXmlDoc, "//Profile/Progress", pdwProgress); } + HRESULT XmlProfileParser::_GetHighPrecisionOutput(IXMLDOMDocument2* pXmlDoc, bool* pfHighPrecisionOutput) { return _GetBool(pXmlDoc, "//Profile/HighPrecisionOutput", pfHighPrecisionOutput); diff --git a/XmlResultParser/XmlResultParser.cpp b/XmlResultParser/XmlResultParser.cpp index f308b1a..d6abc6d 100644 --- a/XmlResultParser/XmlResultParser.cpp +++ b/XmlResultParser/XmlResultParser.cpp @@ -118,7 +118,8 @@ void XmlResultParser::_OutputTargetResults(const TargetResults& results, // TODO: results.readBucketizer; // TODO: results.writeBucketizer; - _OutputValue("Path", results.sPath.c_str()); + _OutputValue("Id", results.iTargetID); + _OutputValue("Path", results.sPath); _OutputValue("BytesCount", results.ullBytesCount); _OutputValue("FileSize", results.ullFileSize); _OutputValue("IOCount", results.ullIOCount); @@ -179,6 +180,8 @@ void XmlResultParser::_OutputLatencySummary(const Histogram& latencyHisto _OutputValue(latencyHistogramName + "LatencyHistogramBins", latencyHistogram.GetBucketCount()); _OutputLatencyInMilliseconds(latencyHistogramName + "Average", latencyHistogram.GetAvg()); _OutputLatencyInMilliseconds(latencyHistogramName + "Stdev", latencyHistogram.GetStandardDeviation()); + _OutputLatencyInMilliseconds(latencyHistogramName + "Min", latencyHistogram.GetMin()); + _OutputLatencyInMilliseconds(latencyHistogramName + "Max", latencyHistogram.GetMax()); } void XmlResultParser::_OutputTargetIops(const IoBucketizer& readBucketizer, @@ -635,6 +638,8 @@ string XmlResultParser::ParseResults(Profile& profile, if (timeSpan.GetMeasureLatency()) { + std::map> targetIDGroups; + Histogram readLatencyHistogram; Histogram writeLatencyHistogram; Histogram totalLatencyHistogram; @@ -643,6 +648,16 @@ string XmlResultParser::ParseResults(Profile& profile, { for (const auto& target : thread.vTargetResults) { + auto it = targetIDGroups.find(target.iTargetID); + if (it != targetIDGroups.end()) + { + (*it).second->Add(target); + } + else + { + targetIDGroups[target.iTargetID] = std::make_shared(target); + } + readLatencyHistogram.Merge(target.readLatencyHistogram); writeLatencyHistogram.Merge(target.writeLatencyHistogram); totalLatencyHistogram.Merge(target.writeLatencyHistogram); @@ -652,10 +667,18 @@ string XmlResultParser::ParseResults(Profile& profile, _OutputLatencySummary(readLatencyHistogram, writeLatencyHistogram, totalLatencyHistogram, profile.GetHistogramBucketList(), fTime); - ConstHistogramBucketListPtr histogramBucketList = profile.GetHistogramBucketList(); - if (histogramBucketList) + if (targetIDGroups.size() > 1) { - _OutputLatencyBuckets(readLatencyHistogram, writeLatencyHistogram, totalLatencyHistogram, histogramBucketList, fTime); + for (const auto& targetIDGroup : targetIDGroups) + { + if (targetIDGroup.second->GetCount() > 1) + { + _Output("\n"); + _OutputTargetResults(*(targetIDGroup.second->GetTargetResults()), timeSpan.GetMeasureLatency(), profile.GetHistogramBucketList(), fTime, + timeSpan.GetCalculateIopsStdDev(), timeSpan.GetIoBucketDurationInMilliseconds()); + _Output("\n"); + } + } } }