Skip to content

Commit

Permalink
Added test for 1 pass 1 fail random failure case (TriBITSPub#600)
Browse files Browse the repository at this point in the history 
Initial test case for 1 passing and 1 failing test in a test history
with identical sha1s between the two, signifying a failing test
  • Loading branch information
@achauphan
achauphan committed Jan 30, 2024
1 parent c966b59 commit d1c94ab
Show file tree
Hide file tree
Showing 6 changed files with 8,623 additions and 3 deletions.
4,216 changes: 4,216 additions & 0 deletions ...lyze_and_report_random_failures/rand_1pass_1fail/test_queries_cache/build_summary_cache/1
View file

Large diffs are not rendered by default.

4,216 changes: 4,216 additions & 0 deletions ...lyze_and_report_random_failures/rand_1pass_1fail/test_queries_cache/build_summary_cache/2
View file

Large diffs are not rendered by default.

74 changes: 74 additions & 0 deletions ...s/rand_1pass_1fail/test_queries_cache/fullCDashNonPassingTests_2018-10-28_2018-10-28.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
{ u'bugtracker': u'https://github.com/trilinos/Trilinos/issues',
u'builds': [ { u'buildName': u'buildname1',
u'buildSummaryLink': u'buildSummary.php?buildid=1323424',
u'buildstarttime': u'2018-10-28T05:56:22 UTC',
u'details': u'Completed (Failed)\n',
u'nprocs': 4,
u'prettyProcTime': u'42s 880ms',
u'prettyTime': u'10s 720ms',
u'procTime': 42.88,
u'site': u'mutrino',
u'siteLink': u'viewSite.php?siteid=223',
u'status': u'Failed',
u'statusclass': u'error',
u'testDetailsLink': u'testDetails.php?test=57860629&build=1323424',
u'testname': u'testname1',
u'time': 10.72}],
u'date': u'2018-10-28',
u'datetime': u'Wednesday, October 31 2018 19:42:20',
u'documentation': u'http://trilinos.org',
u'feed_enabled': False,
u'filterdata': { u'colorblind': 0,
u'debug': 0,
u'filtercombine': u'and',
u'filters': [ { u'compare': 61,
u'field': u'groupname',
u'value': u'Specialized'},
{ u'compare': 65,
u'field': u'buildname',
u'value': u'Trilinos-atdm-'},
{ u'compare': 62,
u'field': u'buildname',
u'value': u'Trilinos-atdm-cee-rhel6-gnu-7.2.0-opt-serial'},
{ u'compare': 62,
u'field': u'buildname',
u'value': u'Trilinos-atdm-white-ride-cuda-9.2-debug-pt'},
{ u'compare': 62,
u'field': u'site',
u'value': u'ride'},
{ u'compare': 62,
u'field': u'status',
u'value': u'passed'}],
u'hasdateclause': 0,
u'limit': 0,
u'pageId': u'queryTests.php',
u'pageSpecificFilters': { u'TextConcat': u"GROUP_CONCAT(text SEPARATOR ', ')"},
u'showfilters': 1,
u'showlimit': 1,
u'sql': u"AND (bg.name = 'Specialized' AND b.name LIKE 'Trilinos-atdm-%' AND b.name != 'Trilinos-atdm-cee-rhel6-gnu-7.2.0-opt-serial' AND b.name != 'Trilinos-atdm-white-ride-cuda-9.2-debug-pt' AND site.name != 'ride' AND build2test.status != 'passed')"},
u'filterurl': u'',
u'footer': u'views/partials/footer.html',
u'generationtime': 0.969,
u'googletracker': u'',
u'hasprocessors': True,
u'header': u'views/partials/header.html',
u'home': u'http://trilinos.org',
u'logoid': 1,
u'menu': { u'back': u'index.php?project=Trilinos&date=2018-10-28',
u'current': u'queryTests.php?project=Trilinos&limit=0',
u'next': u'queryTests.php?project=Trilinos&date=2018-10-29&limit=0',
u'previous': u'queryTests.php?project=Trilinos&date=2018-10-27&limit=0'},
u'nextdate': u'2018-10-29',
u'previousdate': u'2018-10-27',
u'project': { u'showtesttime': 0},
u'projectid': 1,
u'projectname': u'Trilinos',
u'projectname_encoded': u'Trilinos',
u'public': 1,
u'showcalendar': 1,
u'startdate': u'Sunday, October 28 2018 04:01:00',
u'title': u'CDash : Trilinos',
u'unixtimestamp': 1540699260,
u'user': { u'id': 0},
u'vcs': u'http://github.com/trilinos/Trilinos',
u'version': None}
92 changes: 92 additions & 0 deletions ...failures/rand_1pass_1fail/test_queries_cache/test_history_cache/testname1_buildname1.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
{ u'bugtracker': u'https://github.com/trilinos/Trilinos/issues',
u'builds': [ { u'buildName': u'build1',
u'buildSummaryLink': u'build/1',
u'buildstarttime': u'2018-10-28T05:56:22 UTC',
u'details': u'Completed (Failed)\n',
u'nprocs': 4,
u'prettyProcTime': u'42s 880ms',
u'prettyTime': u'10s 720ms',
u'procTime': 42.88,
u'site': u'mutrino',
u'siteLink': u'viewSite.php?siteid=223',
u'status': u'Passed',
u'statusclass': u'error',
u'testDetailsLink': u'testDetails.php?test=57860629&build=1',
u'testname': u'testname1',
u'time': 10.72
},
{ u'buildName': u'build1',
u'buildSummaryLink': u'build/2',
u'buildstarttime': u'2018-10-28T05:56:22 UTC',
u'details': u'Completed (Failed)\n',
u'nprocs': 4,
u'prettyProcTime': u'42s 880ms',
u'prettyTime': u'10s 720ms',
u'procTime': 42.88,
u'site': u'mutrino',
u'siteLink': u'viewSite.php?siteid=223',
u'status': u'Failed',
u'statusclass': u'error',
u'testDetailsLink': u'testDetails.php?test=57860629&build=2',
u'testname': u'testname1',
u'time': 10.72
}
],
u'date': u'2018-10-28',
u'datetime': u'Wednesday, October 31 2018 19:42:20',
u'documentation': u'http://trilinos.org',
u'feed_enabled': False,
u'filterdata': { u'colorblind': 0,
u'debug': 0,
u'filtercombine': u'and',
u'filters': [ { u'compare': 61,
u'field': u'groupname',
u'value': u'Specialized'},
{ u'compare': 65,
u'field': u'buildname',
u'value': u'Trilinos-atdm-'},
{ u'compare': 62,
u'field': u'buildname',
u'value': u'Trilinos-atdm-cee-rhel6-gnu-7.2.0-opt-serial'},
{ u'compare': 62,
u'field': u'buildname',
u'value': u'Trilinos-atdm-white-ride-cuda-9.2-debug-pt'},
{ u'compare': 62,
u'field': u'site',
u'value': u'ride'},
{ u'compare': 62,
u'field': u'status',
u'value': u'passed'}],
u'hasdateclause': 0,
u'limit': 0,
u'pageId': u'queryTests.php',
u'pageSpecificFilters': { u'TextConcat': u"GROUP_CONCAT(text SEPARATOR ', ')"},
u'showfilters': 1,
u'showlimit': 1,
u'sql': u"AND (bg.name = 'Specialized' AND b.name LIKE 'Trilinos-atdm-%' AND b.name != 'Trilinos-atdm-cee-rhel6-gnu-7.2.0-opt-serial' AND b.name != 'Trilinos-atdm-white-ride-cuda-9.2-debug-pt' AND site.name != 'ride' AND build2test.status != 'passed')"},
u'filterurl': u'',
u'footer': u'views/partials/footer.html',
u'generationtime': 0.969,
u'googletracker': u'',
u'hasprocessors': True,
u'header': u'views/partials/header.html',
u'home': u'http://trilinos.org',
u'logoid': 1,
u'menu': { u'back': u'index.php?project=Trilinos&date=2018-10-28',
u'current': u'queryTests.php?project=Trilinos&limit=0',
u'next': u'queryTests.php?project=Trilinos&date=2018-10-29&limit=0',
u'previous': u'queryTests.php?project=Trilinos&date=2018-10-27&limit=0'},
u'nextdate': u'2018-10-29',
u'previousdate': u'2018-10-27',
u'project': { u'showtesttime': 0},
u'projectid': 1,
u'projectname': u'Trilinos',
u'projectname_encoded': u'Trilinos',
u'public': 1,
u'showcalendar': 1,
u'startdate': u'Sunday, October 28 2018 04:01:00',
u'title': u'CDash : Trilinos',
u'unixtimestamp': 1540699260,
u'user': { u'id': 0},
u'vcs': u'http://github.com/trilinos/Trilinos',
u'version': None}
19 changes: 19 additions & 0 deletions test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,25 @@ def test_base(self):
],
)

def test_rand_1pass_1fail(self):
cdash_analyze_and_report_random_failures_setup_test_dir("rand_1pass_1fail")

self.cdash_analyze_and_report_random_failures_run_case(
expectedRtnCode=0,
stdoutRegexList=
[
"[*][*][*] CDash random failure analysis for ProjectName from 2018-10-28 to 2018-10-28",
"Total number of failing tests: 1",

"Found randomly failing tests: 1",
"Test name: testname1",
"Build name: build1",
"Identical sha1 pairs: \(\'592ea0d5\', \'b07e361c\'\)",
"Test history browser URL:",
" https://something[.]com/cdash/queryTests[.]php[?]project=ProjectName&begin=2018-10-28&end=2018-10-28&filtercount=3&showfilters=1&filtercombine=and&field1=testname&compare1=63&value1=testname1&field2=groupname&compare2=63&value2=Pull%20Request&field3=buildname&compare3=63&value3=buildname1"
],
)


if __name__ == "__main__":
unittest.main()
9 changes: 6 additions & 3 deletions tribits/ci_support/cdash_analyze_and_report_random_failures.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,15 +154,18 @@ def main():
buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId)
cache = os.path.join(buildSummaryCacheDir, buildId)
buildConfigOutput = downloadBuildSummaryOffCDash(
buildSummaryQueryUrl, verbose=printUrlMode=='all', alwaysUseCacheFileIfExists=True, buildSummaryCacheFile=cache)['configure']['output']
buildSummaryQueryUrl, verbose=printUrlMode =='all',
alwaysUseCacheFileIfExists=True, buildSummaryCacheFile=cache)['configure']['output']
nonpassingSha1Pairs.add(getTopicTargetSha1s(buildConfigOutput))

# C.2) Check if passing tests' sha1s exist in nonpassing sha1s set
for test in passingTestHistoryLOD:
buildId = getBuildIdFromTest(test)
buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId)
cache = os.path.join(buildSummaryCacheDir, buildId)
buildConfigOutput = downloadBuildSummaryOffCDash(
buildSummaryQueryUrl, verbose=printUrlMode=='all')['configure']['output']
buildSummaryQueryUrl, verbose=printUrlMode =='all',
alwaysUseCacheFileIfExists=True, buildSummaryCacheFile=cache)['configure']['output']
passingSha1Pair = getTopicTargetSha1s(buildConfigOutput)

if checkIfTestUnstable(passingSha1Pair, nonpassingSha1Pairs):
Expand All @@ -183,7 +186,7 @@ def main():
print("Test name: "+summary.testName)
print("Build name: "+summary.buildName)
print("Identical sha1 pairs: "+str(summary.sha1Pair))
print("Test history browser URL: \n"+summary.testHistoryUrl)
print("Test history browser URL: \n "+summary.testHistoryUrl)


def getCmndLineArgs():
Expand Down

0 comments on commit d1c94ab

Please sign in to comment.