Skip to content

Commit

Permalink
Added not random system test (TriBITSPub#600)
Browse files Browse the repository at this point in the history 
Added a not random failure system test case beginning from one failed test
and 5 tests in its history where all tests contain merge commits with
non-matching parents.
  • Loading branch information
@achauphan
achauphan committed Jan 30, 2024
1 parent f442881 commit 3bedc34
Show file tree
Hide file tree
Showing 9 changed files with 21,318 additions and 4 deletions.
4,216 changes: 4,216 additions & 0 deletions ...ilures/not_rand_3pass_2fail/test_queries_cache/build_summary_cache/testname1_buildname1/1
View file

Large diffs are not rendered by default.

4,216 changes: 4,216 additions & 0 deletions ...ilures/not_rand_3pass_2fail/test_queries_cache/build_summary_cache/testname1_buildname1/2
View file

Large diffs are not rendered by default.

4,216 changes: 4,216 additions & 0 deletions ...ilures/not_rand_3pass_2fail/test_queries_cache/build_summary_cache/testname1_buildname1/3
View file

Large diffs are not rendered by default.

4,216 changes: 4,216 additions & 0 deletions ...ilures/not_rand_3pass_2fail/test_queries_cache/build_summary_cache/testname1_buildname1/4
View file

Large diffs are not rendered by default.

4,216 changes: 4,216 additions & 0 deletions ...ilures/not_rand_3pass_2fail/test_queries_cache/build_summary_cache/testname1_buildname1/5
View file

Large diffs are not rendered by default.

74 changes: 74 additions & 0 deletions ...t_rand_3pass_2fail/test_queries_cache/fullCDashNonPassingTests_2018-10-28_2018-10-28.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
{ u'bugtracker': u'https://github.com/trilinos/Trilinos/issues',
u'builds': [ { u'buildName': u'buildname1',
u'buildSummaryLink': u'buildSummary.php?buildid=1323424',
u'buildstarttime': u'2018-10-28T05:56:22 UTC',
u'details': u'Completed (Failed)\n',
u'nprocs': 4,
u'prettyProcTime': u'42s 880ms',
u'prettyTime': u'10s 720ms',
u'procTime': 42.88,
u'site': u'mutrino',
u'siteLink': u'viewSite.php?siteid=223',
u'status': u'Failed',
u'statusclass': u'error',
u'testDetailsLink': u'testDetails.php?test=57860629&build=1323424',
u'testname': u'testname1',
u'time': 10.72}],
u'date': u'2018-10-28',
u'datetime': u'Wednesday, October 31 2018 19:42:20',
u'documentation': u'http://trilinos.org',
u'feed_enabled': False,
u'filterdata': { u'colorblind': 0,
u'debug': 0,
u'filtercombine': u'and',
u'filters': [ { u'compare': 61,
u'field': u'groupname',
u'value': u'Specialized'},
{ u'compare': 65,
u'field': u'buildname',
u'value': u'Trilinos-atdm-'},
{ u'compare': 62,
u'field': u'buildname',
u'value': u'Trilinos-atdm-cee-rhel6-gnu-7.2.0-opt-serial'},
{ u'compare': 62,
u'field': u'buildname',
u'value': u'Trilinos-atdm-white-ride-cuda-9.2-debug-pt'},
{ u'compare': 62,
u'field': u'site',
u'value': u'ride'},
{ u'compare': 62,
u'field': u'status',
u'value': u'passed'}],
u'hasdateclause': 0,
u'limit': 0,
u'pageId': u'queryTests.php',
u'pageSpecificFilters': { u'TextConcat': u"GROUP_CONCAT(text SEPARATOR ', ')"},
u'showfilters': 1,
u'showlimit': 1,
u'sql': u"AND (bg.name = 'Specialized' AND b.name LIKE 'Trilinos-atdm-%' AND b.name != 'Trilinos-atdm-cee-rhel6-gnu-7.2.0-opt-serial' AND b.name != 'Trilinos-atdm-white-ride-cuda-9.2-debug-pt' AND site.name != 'ride' AND build2test.status != 'passed')"},
u'filterurl': u'',
u'footer': u'views/partials/footer.html',
u'generationtime': 0.969,
u'googletracker': u'',
u'hasprocessors': True,
u'header': u'views/partials/header.html',
u'home': u'http://trilinos.org',
u'logoid': 1,
u'menu': { u'back': u'index.php?project=Trilinos&date=2018-10-28',
u'current': u'queryTests.php?project=Trilinos&limit=0',
u'next': u'queryTests.php?project=Trilinos&date=2018-10-29&limit=0',
u'previous': u'queryTests.php?project=Trilinos&date=2018-10-27&limit=0'},
u'nextdate': u'2018-10-29',
u'previousdate': u'2018-10-27',
u'project': { u'showtesttime': 0},
u'projectid': 1,
u'projectname': u'Trilinos',
u'projectname_encoded': u'Trilinos',
u'public': 1,
u'showcalendar': 1,
u'startdate': u'Sunday, October 28 2018 04:01:00',
u'title': u'CDash : Trilinos',
u'unixtimestamp': 1540699260,
u'user': { u'id': 0},
u'vcs': u'http://github.com/trilinos/Trilinos',
u'version': None}
140 changes: 140 additions & 0 deletions ...ures/not_rand_3pass_2fail/test_queries_cache/test_history_cache/testname1_buildname1.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
{ u'bugtracker': u'https://github.com/trilinos/Trilinos/issues',
u'builds': [ { u'buildName': u'build1',
u'buildSummaryLink': u'build/1',
u'buildstarttime': u'2018-10-28T05:56:22 UTC',
u'details': u'Completed\n',
u'nprocs': 4,
u'prettyProcTime': u'42s 880ms',
u'prettyTime': u'10s 720ms',
u'procTime': 42.88,
u'site': u'mutrino',
u'siteLink': u'viewSite.php?siteid=223',
u'status': u'Passed',
u'statusclass': u'error',
u'testDetailsLink': u'testDetails.php?test=57860629&build=1',
u'testname': u'testname1',
u'time': 10.72
},
{ u'buildName': u'build1',
u'buildSummaryLink': u'build/2',
u'buildstarttime': u'2018-10-28T05:56:22 UTC',
u'details': u'Completed (Failed)\n',
u'nprocs': 4,
u'prettyProcTime': u'42s 880ms',
u'prettyTime': u'10s 720ms',
u'procTime': 42.88,
u'site': u'mutrino',
u'siteLink': u'viewSite.php?siteid=223',
u'status': u'Failed',
u'statusclass': u'error',
u'testDetailsLink': u'testDetails.php?test=57860629&build=2',
u'testname': u'testname1',
u'time': 10.72
},
{ u'buildName': u'build1',
u'buildSummaryLink': u'build/3',
u'buildstarttime': u'2018-10-28T05:56:22 UTC',
u'details': u'Completed (Failed)\n',
u'nprocs': 4,
u'prettyProcTime': u'42s 880ms',
u'prettyTime': u'10s 720ms',
u'procTime': 42.88,
u'site': u'mutrino',
u'siteLink': u'viewSite.php?siteid=223',
u'status': u'Failed',
u'statusclass': u'error',
u'testDetailsLink': u'testDetails.php?test=57860629&build=2',
u'testname': u'testname1',
u'time': 10.72
},
{ u'buildName': u'build1',
u'buildSummaryLink': u'build/4',
u'buildstarttime': u'2018-10-28T05:56:22 UTC',
u'details': u'Completed\n',
u'nprocs': 4,
u'prettyProcTime': u'42s 880ms',
u'prettyTime': u'10s 720ms',
u'procTime': 42.88,
u'site': u'mutrino',
u'siteLink': u'viewSite.php?siteid=223',
u'status': u'Passed',
u'statusclass': u'error',
u'testDetailsLink': u'testDetails.php?test=57860629&build=2',
u'testname': u'testname1',
u'time': 10.72
},
{ u'buildName': u'build1',
u'buildSummaryLink': u'build/5',
u'buildstarttime': u'2018-10-28T05:56:22 UTC',
u'details': u'Completed\n',
u'nprocs': 4,
u'prettyProcTime': u'42s 880ms',
u'prettyTime': u'10s 720ms',
u'procTime': 42.88,
u'site': u'mutrino',
u'siteLink': u'viewSite.php?siteid=223',
u'status': u'Passed',
u'statusclass': u'error',
u'testDetailsLink': u'testDetails.php?test=57860629&build=2',
u'testname': u'testname1',
u'time': 10.72
}
],
u'date': u'2018-10-28',
u'datetime': u'Wednesday, October 31 2018 19:42:20',
u'documentation': u'http://trilinos.org',
u'feed_enabled': False,
u'filterdata': { u'colorblind': 0,
u'debug': 0,
u'filtercombine': u'and',
u'filters': [ { u'compare': 61,
u'field': u'groupname',
u'value': u'Specialized'},
{ u'compare': 65,
u'field': u'buildname',
u'value': u'Trilinos-atdm-'},
{ u'compare': 62,
u'field': u'buildname',
u'value': u'Trilinos-atdm-cee-rhel6-gnu-7.2.0-opt-serial'},
{ u'compare': 62,
u'field': u'buildname',
u'value': u'Trilinos-atdm-white-ride-cuda-9.2-debug-pt'},
{ u'compare': 62,
u'field': u'site',
u'value': u'ride'},
{ u'compare': 62,
u'field': u'status',
u'value': u'passed'}],
u'hasdateclause': 0,
u'limit': 0,
u'pageId': u'queryTests.php',
u'pageSpecificFilters': { u'TextConcat': u"GROUP_CONCAT(text SEPARATOR ', ')"},
u'showfilters': 1,
u'showlimit': 1,
u'sql': u"AND (bg.name = 'Specialized' AND b.name LIKE 'Trilinos-atdm-%' AND b.name != 'Trilinos-atdm-cee-rhel6-gnu-7.2.0-opt-serial' AND b.name != 'Trilinos-atdm-white-ride-cuda-9.2-debug-pt' AND site.name != 'ride' AND build2test.status != 'passed')"},
u'filterurl': u'',
u'footer': u'views/partials/footer.html',
u'generationtime': 0.969,
u'googletracker': u'',
u'hasprocessors': True,
u'header': u'views/partials/header.html',
u'home': u'http://trilinos.org',
u'logoid': 1,
u'menu': { u'back': u'index.php?project=Trilinos&date=2018-10-28',
u'current': u'queryTests.php?project=Trilinos&limit=0',
u'next': u'queryTests.php?project=Trilinos&date=2018-10-29&limit=0',
u'previous': u'queryTests.php?project=Trilinos&date=2018-10-27&limit=0'},
u'nextdate': u'2018-10-29',
u'previousdate': u'2018-10-27',
u'project': { u'showtesttime': 0},
u'projectid': 1,
u'projectname': u'Trilinos',
u'projectname_encoded': u'Trilinos',
u'public': 1,
u'showcalendar': 1,
u'startdate': u'Sunday, October 28 2018 04:01:00',
u'title': u'CDash : Trilinos',
u'unixtimestamp': 1540699260,
u'user': { u'id': 0},
u'vcs': u'http://github.com/trilinos/Trilinos',
u'version': None}
20 changes: 20 additions & 0 deletions test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,26 @@ def test_rand_1pass_1fail(self):
],
)

def test_not_rand_3pass_2fail(self):

testCaseName = "not_rand_3pass_2fail"
cdash_analyze_and_report_random_failures_setup_test_dir(testCaseName)

self.cdash_analyze_and_report_random_failures_run_case(
expectedRtnCode=0,
stdoutRegexList=
[
"\s+Test name: testname1",
"\s+Build name: buildname1",
"\s+Size of test history: 5",

"[*][*][*] CDash random failure analysis for ProjectName from 2018-10-28 to 2018-10-28",
"Total number of failing tests: 1",

"Found randomly failing tests: 0",
],
)


if __name__ == "__main__":
unittest.main()
8 changes: 4 additions & 4 deletions tribits/ci_support/cdash_analyze_and_report_random_failures.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ def main():
shortenedBuildName = correctedBuildName[:buildNameMax]

print("\n Getting history from "+dateRangeStart+" to "+dateRangeEnd+" for\n"+\
" test name: "+nonpassingTest['testname']+"\n"+\
" build name: "+correctedBuildName)
" Test name: "+nonpassingTest['testname']+"\n"+\
" Build name: "+correctedBuildName)

cdashTestHistoryFilters = \
"filtercount=3&showfilters=1&filtercombine=and"+\
Expand Down Expand Up @@ -192,14 +192,14 @@ def main():

print("\n*** CDash random failure analysis for "+cdashProjectName+" from " +dateRangeStart+" to "+dateRangeEnd)

print("Total number of failing tests: "+str(len(nonpassingTestsLOD))+"\n")
print("Total number of initial failing tests: "+str(len(nonpassingTestsLOD))+"\n")

print("Found randomly failing tests: "+str(len(randomFailureSummaries)))
for summary in randomFailureSummaries:
print("Test name: "+summary.testName)
print("Build name: "+summary.buildName)
print("Identical sha1 pairs: "+str(summary.sha1Pair))
print("Test history browser URL: \n "+summary.testHistoryUrl)
print("Test history browser URL: \n "+summary.testHistoryUrl+"\n")


def getCmndLineArgs():
Expand Down

0 comments on commit 3bedc34

Please sign in to comment.