diff --git a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py index 1678a0979..b2f8216f6 100644 --- a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py +++ b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py @@ -151,8 +151,8 @@ def cdash_analyze_and_report_random_failures_run_case( htmlFileAbsPath = os.getcwd()+"/"+htmlFileName cmnd = ( - ciSupportDir - + "/cdash_analyze_and_report_random_failures.py" + testCiSupportDir + + "/example_cdash_analyze_and_report_random_failures.py" + " --cdash-project-name='Project Name'" + " --group-name='Group Name'" + " --initial-nonpassing-test-filters='initial_nonpassing_test_filters'" diff --git a/test/ci_support/example_cdash_analyze_and_report_random_failures.py b/test/ci_support/example_cdash_analyze_and_report_random_failures.py new file mode 100755 index 000000000..1d8b7c398 --- /dev/null +++ b/test/ci_support/example_cdash_analyze_and_report_random_failures.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import sys +import argparse +import re as regex + +from FindCISupportDir import * +import CDashAnalyzeReportRandomFailures as CDARRF + + +def main(): + + cdashAnalyzeAndReportRandomFailures = \ + CDARRF.CDashAnalyzeReportRandomFailuresDriver( + ExampleVersionInfoStrategy(), + ExampleExtractBuildNameStrategy()) + + cdashAnalyzeAndReportRandomFailures.runDriver() + + +class ExampleVersionInfoStrategy: + + def getTargetTopicSha1s(self, buildData): + pattern = r"Parent [12]:\n\s+(\w+)" + matchedList = regex.findall(pattern, buildData) + + if len(matchedList) != 2: return None + return tuple(matchedList) + + def checkTargetTopicRandomFailure(self, targetTopicPair, knownTargetTopicPairs): + return targetTopicPair in knownTargetTopicPairs + + +class ExampleExtractBuildNameStrategy: + + def getCoreBuildName(self, fullBuildName): + coreBuildName = fullBuildName.rsplit('-',1)[0] + return coreBuildName + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py new file mode 100644 index 000000000..8a9f39567 --- /dev/null +++ b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py @@ -0,0 +1,314 @@ + +import os +import argparse +import datetime + +from FindGeneralScriptSupport import * +import CDashQueryAnalyzeReport as CDQAR +import cdash_build_testing_date as CBTD + + +class CDashAnalyzeReportRandomFailuresDriver: + + def __init__(self, versionInfoStrategy, extractBuildNameStrategy): + self.versionInfoStrategy = versionInfoStrategy + self.extractBuildNameStrategy = extractBuildNameStrategy + self.args = None + + def runDriver(self): + + self.getCmndLineArgs() + + cdashProjectTestingDayStartTime = "00:00" + # TODO: This should be moved outside in a project specific + # driver script or command line input + cdashSiteUrl = self.args.cdash_site_url + cdashProjectName = self.args.cdash_project_name + initialNonpassingTestFilters = self.args.initial_nonpassing_test_filters + date = self.args.reference_date + groupName = self.args.group_name + daysOfHistory = self.args.days_of_history + printUrlMode = self.args.print_url_mode + writeEmailToFile = self.args.write_email_to_file + sendEmailFrom = self.args.send_email_from + sendEmailTo = self.args.send_email_to + + randomFailureSummaries = [] + + # A.1) Set up date range and directories + + # Construct date range for queryTests filter string + referenceDateDT = CDQAR.convertInputDateArgToYYYYMMDD( + cdashProjectTestingDayStartTime, date) + dateRangeStart, dateRangeEnd = self.getDateRangeTuple(referenceDateDT, daysOfHistory) + dateUrlField = "begin="+dateRangeStart+"&end="+dateRangeEnd + dateRangeStr = dateRangeStart+" to "+dateRangeEnd + + testQueriesCacheDir = os.getcwd()+"/test_queries_cache" + testHistoryCacheDir = testQueriesCacheDir+"/test_history_cache" + createDirsFromPath(testQueriesCacheDir) + createDirsFromPath(testHistoryCacheDir) + + # Construct queryTest.php filter for a date range + initialNonpassingTestQueryFilters = \ + dateUrlField+"&"+initialNonpassingTestFilters + + # A.2) Create starting email body and html string aggregation var + + cdashReportData = CDQAR.CDashReportData() + + cdashReportData.htmlEmailBodyTop += \ + "
\n"
+
+ print("\nGetting list of initial nonpassing tests from CDash from "+dateRangeStr)
+
+ initialNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl(
+ cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters)
+ initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl(
+ cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters)
+
+ if printUrlMode == 'initial' or printUrlMode == 'all':
+ print("\nCDash nonpassing tests browser URL:\n\n"+\
+ " "+initialNonpassingTestBrowserUrl+"\n")
+ print("\nCDash nonpassing tests query URL:\n\n"+\
+ " "+initialNonpassingTestsQueryUrl+"\n")
+
+ initialNonpassingTestsQueryCacheFile = testQueriesCacheDir +\
+ "/initialCDashNonPassingTests_"+dateRangeStart+"_"+dateRangeEnd+".json"
+
+ # List of dictionaries containing the cdash results in rows
+ initialNonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten(
+ initialNonpassingTestsQueryUrl, initialNonpassingTestsQueryCacheFile,\
+ alwaysUseCacheFileIfExists=True)
+
+ cdashReportData.htmlEmailBodyTop += \
+ "" +\
+ "Nonpassing tests scanned on CDash=" +\
+ str(len(initialNonpassingTestsLOD))+"
\n"
+
+ # Ending of scanned details and links paragraph
+ # and start of scanning summaries and table
+ cdashReportData.htmlEmailBodyTop +="
\n"
+
+ # B.2) Get each nonpassing test's testing history
+ for nonpassingTest in initialNonpassingTestsLOD:
+
+ # Remove unique jenkins run ID from build name
+ correctedBuildName = self.extractBuildNameStrategy.getCoreBuildName(nonpassingTest['buildName'])
+
+ buildNameMax = 80
+ shortenedBuildName = correctedBuildName[:buildNameMax]
+
+ print("\n Getting history from "+dateRangeStr+" for\n"+\
+ " Test name: "+nonpassingTest['testname']+"\n"+\
+ " Build name: "+correctedBuildName)
+
+ groupNameNormUrl, = CDQAR.normalizeUrlStrings(groupName)
+
+ testHistoryFilters = \
+ "filtercount=3&showfilters=1&filtercombine=and"+\
+ "&field1=testname&compare1=63&value1="+nonpassingTest['testname']+\
+ "&field2=groupname&compare2=63&value2="+groupNameNormUrl+\
+ "&field3=buildname&compare3=63&value3="+correctedBuildName
+
+ testHistoryQueryFilters = dateUrlField+"&"+testHistoryFilters
+
+ testHistoryCacheFile = testHistoryCacheDir+"/" +\
+ nonpassingTest['testname']+"_"+shortenedBuildName+".json"
+
+ print("\n Creating file to write test history:\n "+testHistoryCacheFile)
+
+ testHistoryQueryUrl = CDQAR.getCDashQueryTestsQueryUrl(
+ cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters)
+ testHistoryBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl(
+ cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters)
+
+ testHistoryLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten(
+ testHistoryQueryUrl, testHistoryCacheFile, alwaysUseCacheFileIfExists=True)
+
+ print("\n Size of test history: "+str(len(testHistoryLOD)))
+
+ if printUrlMode == 'all':
+ print("\n CDash test history browser URL for "+nonpassingTest['testname']+" "+\
+ correctedBuildName+":\n\n"+" "+testHistoryBrowserUrl)
+ print("\n CDash test history query URL for "+nonpassingTest['testname']+" "+\
+ correctedBuildName+":\n\n"+" "+testHistoryQueryUrl)
+
+ if len(testHistoryLOD) < 2:
+ print("\n Size of test history too small for any comparisons, skipping ...\n")
+ continue
+
+ # B.3) Split full testing history to passed and nonpassed lists of dicts
+ passingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Passed"]
+ nonpassingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Failed"]
+ nonpassingSha1Pairs = set()
+
+ print("\n Num of passing tests in test history: "+str(len(passingTestHistoryLOD)))
+ print("\n Num of nonpassing tests in test history: "+str(len(nonpassingTestHistoryLOD)))
+
+ buildSummaryCacheDir = testQueriesCacheDir+"/build_summary_cache/" +\
+ nonpassingTest['testname']+"_"+shortenedBuildName
+ createDirsFromPath(buildSummaryCacheDir)
+ # NOTE: There is an argument to be made that test histories should get their own directory
+ # instead of build summaries and that build summaries should live inside of there
+
+ # C.1) Get all nonpassing tests' sha1s into a set
+ for test in nonpassingTestHistoryLOD:
+
+ buildId = self.getBuildIdFromTest(test)
+
+ buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId
+ buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId)
+ buildConfigOutput = self.downloadBuildSummaryOffCDash(
+ buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all',
+ alwaysUseCacheFileIfExists=True)['configure']['output']
+
+ nonpassingSha1Pairs.add(
+ self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput))
+
+ print("\n Test history failing sha1s: "+str(nonpassingSha1Pairs))
+
+ # C.2) Check if passing tests' sha1s exist in nonpassing sha1s set
+ for test in passingTestHistoryLOD:
+
+ buildId = self.getBuildIdFromTest(test)
+
+ buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId
+ buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId)
+ buildConfigOutput = self.downloadBuildSummaryOffCDash(
+ buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all',
+ alwaysUseCacheFileIfExists=True)['configure']['output']
+
+ passingSha1Pair = \
+ self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput)
+
+ if self.versionInfoStrategy.checkTargetTopicRandomFailure(passingSha1Pair, nonpassingSha1Pairs):
+ print("\n Found passing sha1 pair, " + str(passingSha1Pair)+\
+ " in set of nonpassing sha1 pairs: \n"+str(nonpassingSha1Pairs))
+
+ randomFailureSummaries.append(
+ RandomFailureSummary(test['buildName'], test['testname'],
+ testHistoryBrowserUrl, passingSha1Pair))
+
+
+ print("\n*** CDash random failure analysis for " +\
+ cdashProjectName+" "+groupName+" from " +dateRangeStr)
+
+ print("Total number of initial failing tests: "+str(len(initialNonpassingTestsLOD))+"\n")
+
+ print("Found random failing tests: "+str(len(randomFailureSummaries))+"\n")
+
+ cdashReportData.htmlEmailBodyTop += \
+ "Found random failing tests: "+str(len(randomFailureSummaries))+"
\n"
+
+ if len(randomFailureSummaries) > 0:
+ cdashReportData.globalPass = False
+
+ # Add number of random failing tests 'rft' found to summary data list
+ cdashReportData.summaryLineDataNumbersList.append(
+ "rft="+str(len(randomFailureSummaries)))
+
+ # Add number of initial failing tests 'ift' scanned to summary
+ # data list
+ cdashReportData.summaryLineDataNumbersList.append(
+ "ift="+str(len(initialNonpassingTestsLOD)))
+
+ for summary in randomFailureSummaries:
+ print(str(summary))
+ summary.singleSummaryReporter(cdashReportData)
+
+ summaryLine = CDQAR.getOverallCDashReportSummaryLine(
+ cdashReportData, cdashProjectName+" "+groupName, dateRangeStr)
+ print("\n"+summaryLine)
+
+ # Finish HTML body paragraph
+ cdashReportData.htmlEmailBodyTop += "\n