From cf54c8ea460caac1af4fbbe8bae3036eedd66a3e Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 2 Feb 2024 19:39:59 -0700 Subject: [PATCH] Added email summary report option (#600) Added functionality to build an html file containing analysis results and the ability to report the results via email. --- ...ze_and_report_random_failures_UnitTests.py | 94 ++++++++++++++++--- ...dash_analyze_and_report_random_failures.py | 83 ++++++++++++++-- 2 files changed, 157 insertions(+), 20 deletions(-) diff --git a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py index dbe0d5b05..affadfad3 100644 --- a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py +++ b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py @@ -139,6 +139,7 @@ def cdash_analyze_and_report_random_failures_run_case( self, expectedRtnCode, stdoutRegexList, + htmlFileRegexList, extraCmndLineOptionsList=None, verbose=False, debugPrint=False, @@ -146,6 +147,9 @@ def cdash_analyze_and_report_random_failures_run_case( if not extraCmndLineOptionsList: extraCmndLineOptionsList = [] + htmlFileName = "htmlFile.html" + htmlFileAbsPath = os.getcwd()+"/"+htmlFileName + cmnd = ( ciSupportDir + "/cdash_analyze_and_report_random_failures.py" @@ -153,6 +157,7 @@ def cdash_analyze_and_report_random_failures_run_case( + " --group-name='Group Name'" + " --cdash-site-url='https://something.com/cdash'" + " --reference-date=2018-10-28" + + " --write-email-to-file="+htmlFileName + " " + " ".join(extraCmndLineOptionsList) ) @@ -175,13 +180,20 @@ def cdash_analyze_and_report_random_failures_run_case( self.assertEqual(rtnCode, expectedRtnCode, "Failed with stdout: " + stdout) - assertListOfRegexsFoundInListOfStrs( - self, - stdoutRegexList, - stdout.splitlines(), - stdoutFileAbsPath, - debugPrint=debugPrint, - ) + # Grep stdout for expected list of strings + assertListOfRegexsFoundInListOfStrs(self, stdoutRegexList, + stdout.splitlines(), stdoutFileAbsPath, debugPrint=debugPrint) + + # Grep written HTML file for expected strings + try: + with open(htmlFileName, 'r') as htmlFile: + htmlFileStrList = htmlFile.read().split("\n") + except Exception: + print("WARNING: HTML file not available for this test: "+htmlFileAbsPath) + + assertListOfRegexsFoundInListOfStrs(self, htmlFileRegexList, + htmlFileStrList, htmlFileAbsPath, debugPrint=debugPrint) + def setUp(self): self.test_dir = TemporaryDirectory() @@ -199,10 +211,29 @@ def test_base(self): expectedRtnCode=0, stdoutRegexList=[ "[*][*][*] CDash random failure analysis for Project Name Group Name from 2018-10-28 to 2018-10-28", - "Total number of initial failing tests: 1" + "Total number of initial failing tests: 1", + + "PASSED \(rft=0\): Project Name Group Name on 2018-10-28 to 2018-10-28" ], + htmlFileRegexList=[ + "

PASSED \(rft=0\): Project Name Group Name on 2018-10-28 to 2018-10-28

", + + "

Random test failure scan results for Project Name from 2018-10-28 to 2018-10-28

", + + "

", + "Nonpassing tests scanned on CDash=1
", + "

", + + "

", + "Found random failing tests: 0
", + "

" + ] ) + + # Test the random failure case of a single initial test containing + # one passing and one failing test in its test history, both with the same SHA1 pairs + # def test_rand_1pass_1fail(self): testCaseName = "rand_1pass_1fail" @@ -210,19 +241,40 @@ def test_rand_1pass_1fail(self): self.cdash_analyze_and_report_random_failures_run_case( expectedRtnCode=0, - stdoutRegexList= - [ + stdoutRegexList=[ "[*][*][*] CDash random failure analysis for Project Name Group Name from 2018-10-28 to 2018-10-28", "Total number of initial failing tests: 1", - "Found randomly failing tests: 1", + "Found random failing tests: 1", "Test name: testname1", "Build name: build1", "Identical sha1 pairs: \(\'592ea0d5\', \'b07e361c\'\)", "Test history browser URL:", " https://something[.]com/cdash/queryTests[.]php[?]project=Project%20Name&begin=2018-10-28&end=2018-10-28&filtercount=3&showfilters=1&filtercombine=and&field1=testname&compare1=63&value1=testname1&field2=groupname&compare2=63&value2=Group%20Name&field3=buildname&compare3=63&value3=buildname1", + + "FAILED \(rft=1\): Project Name Group Name on 2018-10-28 to 2018-10-28" + ], + htmlFileRegexList=[ + "

FAILED \(rft=1\): Project Name Group Name on 2018-10-28 to 2018-10-28

", + + "

Random test failure scan results for Project Name from 2018-10-28 to 2018-10-28

", + + "

", + "Nonpassing tests scanned on CDash=1
", + "", + "Found random failing tests: 1
", + "
Build name: build1", + "
Test name: testname1", + "
Test history URL: https://something[.]com/cdash/queryTests[.]php[?]project=Project%20Name&begin=2018-10-28&end=2018-10-28&filtercount=3&showfilters=1&filtercombine=and&field1=testname&compare1=63&value1=testname1&field2=groupname&compare2=63&value2=Group%20Name&field3=buildname&compare3=63&value3=buildname1", + "
Sha1 Pair : \('592ea0d5', 'b07e361c'\)", + "

" ], + extraCmndLineOptionsList=[ + ] ) + # TODO: Remove the current "random" test case as this should be the base case def test_not_rand_3pass_2fail(self): @@ -231,8 +283,7 @@ def test_not_rand_3pass_2fail(self): self.cdash_analyze_and_report_random_failures_run_case( expectedRtnCode=0, - stdoutRegexList= - [ + stdoutRegexList=[ "\s+Test name: testname1", "\s+Build name: buildname1", "\s+Size of test history: 5", @@ -240,8 +291,23 @@ def test_not_rand_3pass_2fail(self): "[*][*][*] CDash random failure analysis for Project Name Group Name from 2018-10-28 to 2018-10-28", "Total number of initial failing tests: 1", - "Found randomly failing tests: 0", + "Found random failing tests: 0", + + "PASSED \(rft=0\): Project Name Group Name on 2018-10-28 to 2018-10-28" ], + htmlFileRegexList=[ + "

PASSED \(rft=0\): Project Name Group Name on 2018-10-28 to 2018-10-28

", + + "

Random test failure scan results for Project Name from 2018-10-28 to 2018-10-28

", + + "

", + "Nonpassing tests scanned on CDash=1
", + "

", + + "

", + "Found random failing tests: 0
", + "

" + ] ) diff --git a/tribits/ci_support/cdash_analyze_and_report_random_failures.py b/tribits/ci_support/cdash_analyze_and_report_random_failures.py index 8438a43a5..082a263db 100755 --- a/tribits/ci_support/cdash_analyze_and_report_random_failures.py +++ b/tribits/ci_support/cdash_analyze_and_report_random_failures.py @@ -45,6 +45,13 @@ def __str__(self): "\n "+self.testHistoryUrl+"\n" return myStr + def singleSummaryReporter(self, cdashReportData): + cdashReportData.htmlEmailBodyTop += \ + "\n
Build name: "+ self.buildName +\ + "\n
Test name: "+ self.testName +\ + "\n
Test history URL: "+ self.testHistoryUrl +\ + "\n
Sha1 Pair : "+ str(self.sha1Pair) + # The main function def main(): @@ -60,6 +67,9 @@ def main(): groupName = args.group_name daysOfHistory = args.days_of_history printUrlMode = args.print_url_mode + writeEmailToFile = args.write_email_to_file + sendEmailFrom = args.send_email_from + sendEmailTo = args.send_email_to randomFailureSummaries = [] @@ -71,13 +81,14 @@ def main(): # and into a project-specific driver script or taken as a # command line input. - # A) Set up date range and directories + # A.1) Set up date range and directories # Construct date range for queryTests filter string referenceDateDT = CDQAR.convertInputDateArgToYYYYMMDD( cdashProjectTestingDayStartTime, date) dateRangeStart, dateRangeEnd = getDateRangeTuple(referenceDateDT, daysOfHistory) dateUrlField = "begin="+dateRangeStart+"&end="+dateRangeEnd + dateRangeStr = dateRangeStart+" to "+dateRangeEnd print("\n dateRangeBeginStr: "+dateRangeStart+" dateRangeEndStr: "+dateRangeEnd) @@ -90,16 +101,27 @@ def main(): initialNonpassingTestQueryFilters = \ dateUrlField+"&"+cdashInitialNonpassedTestsFilters + # A.2) Create starting email body and html string aggregation var + + cdashReportData = CDQAR.CDashReportData() + + cdashReportData.htmlEmailBodyTop += \ + "

Random test failure scan results for "+cdashProjectName\ + +" from "+dateRangeStr+"

\n\n" + # B.1) Get all failing test result for past daysOfHistory + # Beginning of scanned details and links paragraph + cdashReportData.htmlEmailBodyTop +="

\n" + print("\nGetting list of nonpassing tests from CDash ...") initialNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) + initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( + cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) if printUrlMode == 'initial' or printUrlMode == 'all': - initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( - cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) print("\nCDash nonpassing tests browser URL:\n\n"+\ " "+initialNonpassingTestBrowserUrl+"\n") print("\nCDash nonpassing tests query URL:\n\n"+\ @@ -113,6 +135,15 @@ def main(): initialNonpassingTestsQueryUrl, initialNonpassingTestsQueryCacheFile,\ alwaysUseCacheFileIfExists=True) + cdashReportData.htmlEmailBodyTop += \ + "" +\ + "Nonpassing tests scanned on CDash=" +\ + str(len(initialNonpassingTestsLOD))+"
\n" + + # Ending of scanned details and links paragraph + # and start of scanning summaries and table + cdashReportData.htmlEmailBodyTop +="

\n\n

\n" + # B.2) Get each nonpassing test's testing history for nonpassingTest in initialNonpassingTestsLOD: @@ -125,7 +156,7 @@ def main(): buildNameMax = 80 shortenedBuildName = correctedBuildName[:buildNameMax] - print("\n Getting history from "+dateRangeStart+" to "+dateRangeEnd+" for\n"+\ + print("\n Getting history from "+dateRangeStr+" for\n"+\ " Test name: "+nonpassingTest['testname']+"\n"+\ " Build name: "+correctedBuildName) @@ -217,9 +248,47 @@ def main(): print("Total number of initial failing tests: "+str(len(initialNonpassingTestsLOD))+"\n") - print("Found randomly failing tests: "+str(len(randomFailureSummaries))) + print("Found random failing tests: "+str(len(randomFailureSummaries))+"\n") + + cdashReportData.htmlEmailBodyTop += \ + "Found random failing tests: "+str(len(randomFailureSummaries))+"
\n" + + if len(randomFailureSummaries) > 0: + cdashReportData.globalPass = False + + cdashReportData.summaryLineDataNumbersList.append( + "rft="+str(len(randomFailureSummaries))) + for summary in randomFailureSummaries: print(str(summary)) + summary.singleSummaryReporter(cdashReportData) + + summaryLine = CDQAR.getOverallCDashReportSummaryLine( + cdashReportData, cdashProjectName+" "+groupName, dateRangeStr) + print("\n"+summaryLine) + + # Finish HTML body paragraph + cdashReportData.htmlEmailBodyTop += "\n

" + + if writeEmailToFile: + print("\nWriting HTML to file: "+writeEmailToFile+" ...") + defaultPageStyle = CDQAR.getDefaultHtmlPageStyleStr() + htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, + pageTitle=summaryLine, pageStyle=defaultPageStyle) + # print(htmlStr) + with open(writeEmailToFile, 'w') as outFile: + outFile.write(htmlStr) + + if sendEmailTo: + htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, + pageStyle=defaultPageStyle) + for emailAddress in sendEmailTo.split(','): + emailAddress = emailAddress.strip() + print("\nSending email to '"+emailAddress+"' ...") + msg=CDQAR.createHtmlMimeEmail( + sendEmailFrom, emailAddress, summaryLine, "", + htmlStr) + CDQAR.sendMineEmail(msg) @@ -231,11 +300,13 @@ def getCmndLineArgs(): parser.add_argument("--group-name", default="Pull Request") parser.add_argument("--days-of-history", default=1, type=int) parser.add_argument("--print-url-mode", choices=['none','initial','all'], default='none') + parser.add_argument("--write-email-to-file", default="") + parser.add_argument("--send-email-to", default="") + parser.add_argument("--send-email-from", default="random-failure-script@noreply.org") return parser.parse_args() - def getDateRangeTuple(referenceDateTime, dayTimeDelta): beginDateTime = referenceDateTime - datetime.timedelta(days=(dayTimeDelta-1)) beginDateTimeStr = CBTD.getDateStrFromDateTime(beginDateTime)