Added email summary report option (TriBITSPub#600)

Added functionality to build an html file containing analysis results
and the ability to report the results via email.

test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py

@@ -139,20 +139,25 @@ def cdash_analyze_and_report_random_failures_run_case(
         self,
         expectedRtnCode,
         stdoutRegexList,
+        htmlFileRegexList,
         extraCmndLineOptionsList=None,
         verbose=False,
         debugPrint=False,
     ):
         if not extraCmndLineOptionsList:
             extraCmndLineOptionsList = []
 
+        htmlFileName = "htmlFile.html"
+        htmlFileAbsPath = os.getcwd()+"/"+htmlFileName
+
         cmnd = (
             ciSupportDir
             + "/cdash_analyze_and_report_random_failures.py"
             + " --cdash-project-name='Project Name'"
             + " --group-name='Group Name'"
             + " --cdash-site-url='https://something.com/cdash'"
             + " --reference-date=2018-10-28"
+            + " --write-email-to-file="+htmlFileName
             + " "
             + " ".join(extraCmndLineOptionsList)
         )
@@ -175,13 +180,20 @@ def cdash_analyze_and_report_random_failures_run_case(
 
         self.assertEqual(rtnCode, expectedRtnCode, "Failed with stdout: " + stdout)
 
-        assertListOfRegexsFoundInListOfStrs(
-            self,
-            stdoutRegexList,
-            stdout.splitlines(),
-            stdoutFileAbsPath,
-            debugPrint=debugPrint,
-        )
+        # Grep stdout for expected list of strings
+        assertListOfRegexsFoundInListOfStrs(self, stdoutRegexList,
+            stdout.splitlines(), stdoutFileAbsPath, debugPrint=debugPrint)
+
+        # Grep written HTML file for expected strings
+        try:
+            with open(htmlFileName, 'r') as htmlFile:
+                htmlFileStrList = htmlFile.read().split("\n")
+        except Exception:
+            print("WARNING: HTML file not available for this test: "+htmlFileAbsPath)
+
+        assertListOfRegexsFoundInListOfStrs(self, htmlFileRegexList,
+            htmlFileStrList, htmlFileAbsPath, debugPrint=debugPrint)
+
 
     def setUp(self):
         self.test_dir = TemporaryDirectory()
@@ -199,30 +211,70 @@ def test_base(self):
             expectedRtnCode=0,
             stdoutRegexList=[
                 "[*][*][*] CDash random failure analysis for Project Name Group Name from 2018-10-28 to 2018-10-28",
-                "Total number of initial failing tests: 1"
+                "Total number of initial failing tests: 1",
+
+                "PASSED \(rft=0\): Project Name Group Name on 2018-10-28 to 2018-10-28"
             ],
+            htmlFileRegexList=[
+                "<h2>PASSED \(rft=0\): Project Name Group Name on 2018-10-28 to 2018-10-28</h2>",
+
+                "<h2>Random test failure scan results for Project Name from 2018-10-28 to 2018-10-28</h2>",
+
+                "<p>",
+                "<a href=\"https://something[.]com/cdash/queryTests[.]php[?]project=Project%20Name&begin=2018-10-28&end=2018-10-28&filtercount=2&showfilters=1&filtercombine=and&field1=status&compare1=63&value1=Failed&field2=groupname&compare2=63&value2=Pull%20Request\">Nonpassing tests scanned on CDash</a>=1<br>",
+                "</p>",
+
+                "<p>",
+                "Found random failing tests: 0<br>",
+                "</p>"
+            ]
         )
 
+
+    # Test the random failure case of a single initial test containing
+    # one passing and one failing test in its test history, both with the same SHA1 pairs
+    #
     def test_rand_1pass_1fail(self):
 
         testCaseName = "rand_1pass_1fail"
         cdash_analyze_and_report_random_failures_setup_test_dir(testCaseName)
 
         self.cdash_analyze_and_report_random_failures_run_case(
             expectedRtnCode=0,
-            stdoutRegexList=
-            [
+            stdoutRegexList=[
                 "[*][*][*] CDash random failure analysis for Project Name Group Name from 2018-10-28 to 2018-10-28",
                 "Total number of initial failing tests: 1",
 
-                "Found randomly failing tests: 1",
+                "Found random failing tests: 1",
                 "Test name: testname1",
                 "Build name: build1",
                 "Identical sha1 pairs: \(\'592ea0d5\', \'b07e361c\'\)",
                 "Test history browser URL:",
                 "  https://something[.]com/cdash/queryTests[.]php[?]project=Project%20Name&begin=2018-10-28&end=2018-10-28&filtercount=3&showfilters=1&filtercombine=and&field1=testname&compare1=63&value1=testname1&field2=groupname&compare2=63&value2=Group%20Name&field3=buildname&compare3=63&value3=buildname1",
+
+                "FAILED \(rft=1\): Project Name Group Name on 2018-10-28 to 2018-10-28"
+            ],
+            htmlFileRegexList=[
+                "<h2>FAILED \(rft=1\): Project Name Group Name on 2018-10-28 to 2018-10-28</h2>",
+
+                "<h2>Random test failure scan results for Project Name from 2018-10-28 to 2018-10-28</h2>",
+
+                "<p>",
+                "<a href=\"https://something[.]com/cdash/queryTests[.]php[?]project=Project%20Name&begin=2018-10-28&end=2018-10-28&filtercount=2&showfilters=1&filtercombine=and&field1=status&compare1=63&value1=Failed&field2=groupname&compare2=63&value2=Pull%20Request\">Nonpassing tests scanned on CDash</a>=1<br>",
+                "</p",
+
+                "<p>",
+                "Found random failing tests: 1<br>",
+                "<br>Build name: build1",
+                "<br>Test name: testname1",
+                "<br>Test history URL: https://something[.]com/cdash/queryTests[.]php[?]project=Project%20Name&begin=2018-10-28&end=2018-10-28&filtercount=3&showfilters=1&filtercombine=and&field1=testname&compare1=63&value1=testname1&field2=groupname&compare2=63&value2=Group%20Name&field3=buildname&compare3=63&value3=buildname1",
+                "<br>Sha1 Pair : \('592ea0d5', 'b07e361c'\)",
+                "</p>"
             ],
+            extraCmndLineOptionsList=[
+            ]
         )
+    # TODO: Remove the current "random" test case as this should be the base case
 
     def test_not_rand_3pass_2fail(self):
 
@@ -231,17 +283,31 @@ def test_not_rand_3pass_2fail(self):
 
         self.cdash_analyze_and_report_random_failures_run_case(
             expectedRtnCode=0,
-            stdoutRegexList=
-            [
+            stdoutRegexList=[
                 "\s+Test name: testname1",
                 "\s+Build name: buildname1",
                 "\s+Size of test history: 5",
 
                 "[*][*][*] CDash random failure analysis for Project Name Group Name from 2018-10-28 to 2018-10-28",
                 "Total number of initial failing tests: 1",
 
-                "Found randomly failing tests: 0",
+                "Found random failing tests: 0",
+
+                "PASSED \(rft=0\): Project Name Group Name on 2018-10-28 to 2018-10-28"
             ],
+            htmlFileRegexList=[
+                "<h2>PASSED \(rft=0\): Project Name Group Name on 2018-10-28 to 2018-10-28</h2>",
+
+                "<h2>Random test failure scan results for Project Name from 2018-10-28 to 2018-10-28</h2>",
+
+                "<p>",
+                "<a href=\"https://something[.]com/cdash/queryTests[.]php[?]project=Project%20Name&begin=2018-10-28&end=2018-10-28&filtercount=2&showfilters=1&filtercombine=and&field1=status&compare1=63&value1=Failed&field2=groupname&compare2=63&value2=Pull%20Request\">Nonpassing tests scanned on CDash</a>=1<br>",
+                "</p>",
+
+                "<p>",
+                "Found random failing tests: 0<br>",
+                "</p>"
+            ]
         )
 
 

tribits/ci_support/cdash_analyze_and_report_random_failures.py

@@ -45,6 +45,13 @@ def __str__(self):
       "\n  "+self.testHistoryUrl+"\n"
     return myStr
 
+  def singleSummaryReporter(self, cdashReportData):
+    cdashReportData.htmlEmailBodyTop += \
+      "\n<br>Build name: "+ self.buildName +\
+      "\n<br>Test name: "+ self.testName +\
+      "\n<br>Test history URL: "+ self.testHistoryUrl +\
+      "\n<br>Sha1 Pair : "+ str(self.sha1Pair)
+
 
 # The main function
 def main():
@@ -60,6 +67,9 @@ def main():
   groupName = args.group_name
   daysOfHistory = args.days_of_history
   printUrlMode = args.print_url_mode
+  writeEmailToFile = args.write_email_to_file
+  sendEmailFrom = args.send_email_from
+  sendEmailTo = args.send_email_to
 
   randomFailureSummaries = []
 
@@ -71,13 +81,14 @@ def main():
   # and into a project-specific driver script or taken as a
   # command line input.
 
-  # A) Set up date range and directories
+  # A.1) Set up date range and directories
 
   # Construct date range for queryTests filter string
   referenceDateDT = CDQAR.convertInputDateArgToYYYYMMDD(
     cdashProjectTestingDayStartTime, date)
   dateRangeStart, dateRangeEnd = getDateRangeTuple(referenceDateDT, daysOfHistory)
   dateUrlField = "begin="+dateRangeStart+"&end="+dateRangeEnd
+  dateRangeStr = dateRangeStart+" to "+dateRangeEnd
 
   print("\n dateRangeBeginStr: "+dateRangeStart+" dateRangeEndStr: "+dateRangeEnd)
 
@@ -90,16 +101,27 @@ def main():
   initialNonpassingTestQueryFilters = \
     dateUrlField+"&"+cdashInitialNonpassedTestsFilters
 
+  # A.2) Create starting email body and html string aggregation var
+
+  cdashReportData = CDQAR.CDashReportData()
+
+  cdashReportData.htmlEmailBodyTop += \
+    "<h2>Random test failure scan results for "+cdashProjectName\
+      +" from "+dateRangeStr+"</h2>\n\n"
+
   # B.1) Get all failing test result for past daysOfHistory
 
+  # Beginning of scanned details and links paragraph
+  cdashReportData.htmlEmailBodyTop +="<p>\n"
+
   print("\nGetting list of nonpassing tests from CDash ...")
 
   initialNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl(
     cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters)
+  initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl(
+    cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters)
 
   if printUrlMode == 'initial' or printUrlMode == 'all':
-    initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl(
-      cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters)
     print("\nCDash nonpassing tests browser URL:\n\n"+\
       "  "+initialNonpassingTestBrowserUrl+"\n")
     print("\nCDash nonpassing tests query URL:\n\n"+\
@@ -113,6 +135,15 @@ def main():
     initialNonpassingTestsQueryUrl, initialNonpassingTestsQueryCacheFile,\
     alwaysUseCacheFileIfExists=True)
 
+  cdashReportData.htmlEmailBodyTop += \
+    "<a href=\""+initialNonpassingTestBrowserUrl+"\">" +\
+    "Nonpassing tests scanned on CDash</a>=" +\
+    str(len(initialNonpassingTestsLOD))+"<br>\n"
+
+  # Ending of scanned details and links paragraph
+  # and start of scanning summaries and table
+  cdashReportData.htmlEmailBodyTop +="</p>\n\n<p>\n"
+
   # B.2) Get each nonpassing test's testing history
   for nonpassingTest in initialNonpassingTestsLOD:
 
@@ -125,7 +156,7 @@ def main():
     buildNameMax = 80
     shortenedBuildName = correctedBuildName[:buildNameMax]
 
-    print("\n Getting history from "+dateRangeStart+" to "+dateRangeEnd+" for\n"+\
+    print("\n Getting history from "+dateRangeStr+" for\n"+\
           "  Test name: "+nonpassingTest['testname']+"\n"+\
           "  Build name: "+correctedBuildName)
 
@@ -217,9 +248,47 @@ def main():
 
   print("Total number of initial failing tests: "+str(len(initialNonpassingTestsLOD))+"\n")
 
-  print("Found randomly failing tests: "+str(len(randomFailureSummaries)))
+  print("Found random failing tests: "+str(len(randomFailureSummaries))+"\n")
+
+  cdashReportData.htmlEmailBodyTop += \
+    "Found random failing tests: "+str(len(randomFailureSummaries))+"<br>\n"
+
+  if len(randomFailureSummaries) > 0:
+    cdashReportData.globalPass = False
+
+  cdashReportData.summaryLineDataNumbersList.append(
+    "rft="+str(len(randomFailureSummaries)))
+
   for summary in randomFailureSummaries:
     print(str(summary))
+    summary.singleSummaryReporter(cdashReportData)
+
+  summaryLine = CDQAR.getOverallCDashReportSummaryLine(
+    cdashReportData, cdashProjectName+" "+groupName, dateRangeStr)
+  print("\n"+summaryLine)
+
+  # Finish HTML body paragraph
+  cdashReportData.htmlEmailBodyTop += "\n</p>"
+
+  if writeEmailToFile:
+    print("\nWriting HTML to file: "+writeEmailToFile+" ...")
+    defaultPageStyle = CDQAR.getDefaultHtmlPageStyleStr()
+    htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData,
+      pageTitle=summaryLine, pageStyle=defaultPageStyle)
+    # print(htmlStr)
+    with open(writeEmailToFile, 'w') as outFile:
+      outFile.write(htmlStr)
+
+  if sendEmailTo:
+    htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData,
+    pageStyle=defaultPageStyle)
+    for emailAddress in sendEmailTo.split(','):
+      emailAddress = emailAddress.strip()
+      print("\nSending email to '"+emailAddress+"' ...")
+      msg=CDQAR.createHtmlMimeEmail(
+        sendEmailFrom, emailAddress, summaryLine, "",
+        htmlStr)
+      CDQAR.sendMineEmail(msg)
 
 
 
@@ -231,11 +300,13 @@ def getCmndLineArgs():
   parser.add_argument("--group-name", default="Pull Request")
   parser.add_argument("--days-of-history", default=1, type=int)
   parser.add_argument("--print-url-mode", choices=['none','initial','all'], default='none')
+  parser.add_argument("--write-email-to-file", default="")
+  parser.add_argument("--send-email-to", default="")
+  parser.add_argument("--send-email-from", default="[email protected]")
 
   return parser.parse_args()
 
 
-
 def getDateRangeTuple(referenceDateTime, dayTimeDelta):
   beginDateTime = referenceDateTime - datetime.timedelta(days=(dayTimeDelta-1))
   beginDateTimeStr = CBTD.getDateStrFromDateTime(beginDateTime)