From eb6f87e04eac5b6bfa03a91194df7d9dc4603058 Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 9 Feb 2024 12:57:01 -0700 Subject: [PATCH] Copy cdash_analyze_and_report_random_failures.py main into module file (#600) This large commit is copying over the main() function and its associated helper functions into CDashAnalyzeReportRandomFailures.py inside the CDashAnalyzeReportRandomFailuresDriver class. This is part of the effort to refactor cdash_analyze_and_report_random_failures.py to be more generic. --- ...ze_and_report_random_failures_UnitTests.py | 2 +- .../CDashAnalyzeReportRandomFailures.py | 307 +++++++++++++++++- ...dash_analyze_and_report_random_failures.py | 12 +- 3 files changed, 313 insertions(+), 8 deletions(-) diff --git a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py index 1678a0979..a7d0ccfb9 100644 --- a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py +++ b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py @@ -152,7 +152,7 @@ def cdash_analyze_and_report_random_failures_run_case( cmnd = ( ciSupportDir - + "/cdash_analyze_and_report_random_failures.py" + + "/example_cdash_analyze_and_report_random_failures.py" + " --cdash-project-name='Project Name'" + " --group-name='Group Name'" + " --initial-nonpassing-test-filters='initial_nonpassing_test_filters'" diff --git a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py index c9346c2bb..52add5887 100644 --- a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py +++ b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py @@ -1,16 +1,317 @@ import os import argparse +import datetime from FindGeneralScriptSupport import * import CDashQueryAnalyzeReport as CDQAR +import cdash_build_testing_date as CBTD + class CDashAnalyzeReportRandomFailuresDriver: def __init__(self, versionInfoStrategy, extractBuildNameStrategy): - self.getTargetTopicSha1Strategy = versionInfoStrategy - self.getextractBuildNameStrategy = extractBuildNameStrategy + self.versionInfoStrategy = versionInfoStrategy + self.extractBuildNameStrategy = extractBuildNameStrategy self.args = None def runDriver(self): - pass + + self.getCmndLineArgs() + + cdashProjectTestingDayStartTime = "00:00" + # TODO: This should be moved outside in a project specific + # driver script or command line input + cdashSiteUrl = self.args.cdash_site_url + cdashProjectName = self.args.cdash_project_name + initialNonpassingTestFilters = self.args.initial_nonpassing_test_filters + date = self.args.reference_date + groupName = self.args.group_name + daysOfHistory = self.args.days_of_history + printUrlMode = self.args.print_url_mode + writeEmailToFile = self.args.write_email_to_file + sendEmailFrom = self.args.send_email_from + sendEmailTo = self.args.send_email_to + + randomFailureSummaries = [] + + # A.1) Set up date range and directories + + # Construct date range for queryTests filter string + referenceDateDT = CDQAR.convertInputDateArgToYYYYMMDD( + cdashProjectTestingDayStartTime, date) + dateRangeStart, dateRangeEnd = self.getDateRangeTuple(referenceDateDT, daysOfHistory) + dateUrlField = "begin="+dateRangeStart+"&end="+dateRangeEnd + dateRangeStr = dateRangeStart+" to "+dateRangeEnd + + testQueriesCacheDir = os.getcwd()+"/test_queries_cache" + testHistoryCacheDir = testQueriesCacheDir+"/test_history_cache" + createDirsFromPath(testQueriesCacheDir) + createDirsFromPath(testHistoryCacheDir) + + # Construct queryTest.php filter for a date range + initialNonpassingTestQueryFilters = \ + dateUrlField+"&"+initialNonpassingTestFilters + + # A.2) Create starting email body and html string aggregation var + + cdashReportData = CDQAR.CDashReportData() + + cdashReportData.htmlEmailBodyTop += \ + "

Random test failure scan results for "+cdashProjectName\ + +" from "+dateRangeStr+"

\n\n" + + + # B.1) Get all failing test result for past daysOfHistory + + # Beginning of scanned details and links paragraph + cdashReportData.htmlEmailBodyTop +="

\n" + + print("\nGetting list of initial nonpassing tests from CDash from "+dateRangeStr) + + initialNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( + cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) + initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( + cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) + + if printUrlMode == 'initial' or printUrlMode == 'all': + print("\nCDash nonpassing tests browser URL:\n\n"+\ + " "+initialNonpassingTestBrowserUrl+"\n") + print("\nCDash nonpassing tests query URL:\n\n"+\ + " "+initialNonpassingTestsQueryUrl+"\n") + + initialNonpassingTestsQueryCacheFile = testQueriesCacheDir +\ + "/initialCDashNonPassingTests_"+dateRangeStart+"_"+dateRangeEnd+".json" + + # List of dictionaries containing the cdash results in rows + initialNonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( + initialNonpassingTestsQueryUrl, initialNonpassingTestsQueryCacheFile,\ + alwaysUseCacheFileIfExists=True) + + cdashReportData.htmlEmailBodyTop += \ + "" +\ + "Nonpassing tests scanned on CDash=" +\ + str(len(initialNonpassingTestsLOD))+"
\n" + + # Ending of scanned details and links paragraph + # and start of scanning summaries and table + cdashReportData.htmlEmailBodyTop +="

\n\n

\n" + + # B.2) Get each nonpassing test's testing history + for nonpassingTest in initialNonpassingTestsLOD: + + # Remove unique jenkins run ID from build name + correctedBuildName = nonpassingTest['buildName'].rsplit('-', 1)[0] + # NOTE: This is project specific code. As Ross pointed out, buildName + # contains a lot of Trilinos specific prefix and suffix that should + # be processed by a Strategy class to be more project agnostic. + + buildNameMax = 80 + shortenedBuildName = correctedBuildName[:buildNameMax] + + print("\n Getting history from "+dateRangeStr+" for\n"+\ + " Test name: "+nonpassingTest['testname']+"\n"+\ + " Build name: "+correctedBuildName) + + groupNameNormUrl, = CDQAR.normalizeUrlStrings(groupName) + + testHistoryFilters = \ + "filtercount=3&showfilters=1&filtercombine=and"+\ + "&field1=testname&compare1=63&value1="+nonpassingTest['testname']+\ + "&field2=groupname&compare2=63&value2="+groupNameNormUrl+\ + "&field3=buildname&compare3=63&value3="+correctedBuildName + + testHistoryQueryFilters = dateUrlField+"&"+testHistoryFilters + + testHistoryCacheFile = testHistoryCacheDir+"/" +\ + nonpassingTest['testname']+"_"+shortenedBuildName+".json" + + print("\n Creating file to write test history:\n "+testHistoryCacheFile) + + testHistoryQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( + cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters) + testHistoryBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( + cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters) + + testHistoryLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( + testHistoryQueryUrl, testHistoryCacheFile, alwaysUseCacheFileIfExists=True) + + print("\n Size of test history: "+str(len(testHistoryLOD))) + + if printUrlMode == 'all': + print("\n CDash test history browser URL for "+nonpassingTest['testname']+" "+\ + correctedBuildName+":\n\n"+" "+testHistoryBrowserUrl) + print("\n CDash test history query URL for "+nonpassingTest['testname']+" "+\ + correctedBuildName+":\n\n"+" "+testHistoryQueryUrl) + + if len(testHistoryLOD) < 2: + print("\n Size of test history too small for any comparisons, skipping ...\n") + continue + + # B.3) Split full testing history to passed and nonpassed lists of dicts + passingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Passed"] + nonpassingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Failed"] + nonpassingSha1Pairs = set() + + print("\n Num of passing tests in test history: "+str(len(passingTestHistoryLOD))) + print("\n Num of nonpassing tests in test history: "+str(len(nonpassingTestHistoryLOD))) + + buildSummaryCacheDir = testQueriesCacheDir+"/build_summary_cache/" +\ + nonpassingTest['testname']+"_"+shortenedBuildName + createDirsFromPath(buildSummaryCacheDir) + # NOTE: There is an argument to be made that test histories should get their own directory + # instead of build summaries and that build summaries should live inside of there + + # C.1) Get all nonpassing tests' sha1s into a set + for test in nonpassingTestHistoryLOD: + + buildId = self.getBuildIdFromTest(test) + + buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId + buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId) + buildConfigOutput = self.downloadBuildSummaryOffCDash( + buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all', + alwaysUseCacheFileIfExists=True)['configure']['output'] + + nonpassingSha1Pairs.add( + self.versionInfoStrategy.getTopicTargetSha1s(buildConfigOutput)) + # nonpassingSha1Pairs.add(getTopicTargetSha1s(buildConfigOutput)) + + print("\n Test history failing sha1s: "+str(nonpassingSha1Pairs)) + + # C.2) Check if passing tests' sha1s exist in nonpassing sha1s set + for test in passingTestHistoryLOD: + + buildId = self.getBuildIdFromTest(test) + + buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId + buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId) + buildConfigOutput = self.downloadBuildSummaryOffCDash( + buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all', + alwaysUseCacheFileIfExists=True)['configure']['output'] + + passingSha1Pair = \ + self.versionInfoStrategy.getTopicTargetSha1s(buildConfigOutput) + + if self.versionInfoStrategy.checkTargetTopicRandomFailure(passingSha1Pair, nonpassingSha1Pairs): + print("\n Found passing sha1 pair, " + str(passingSha1Pair)+\ + " in set of nonpassing sha1 pairs: \n"+str(nonpassingSha1Pairs)) + + randomFailureSummaries.append( + RandomFailureSummary(test['buildName'], test['testname'], + testHistoryBrowserUrl, passingSha1Pair)) + + + print("\n*** CDash random failure analysis for " +\ + cdashProjectName+" "+groupName+" from " +dateRangeStr) + + print("Total number of initial failing tests: "+str(len(initialNonpassingTestsLOD))+"\n") + + print("Found random failing tests: "+str(len(randomFailureSummaries))+"\n") + + cdashReportData.htmlEmailBodyTop += \ + "Found random failing tests: "+str(len(randomFailureSummaries))+"
\n" + + if len(randomFailureSummaries) > 0: + cdashReportData.globalPass = False + + # Add number of random failing tests 'rft' found to summary data list + cdashReportData.summaryLineDataNumbersList.append( + "rft="+str(len(randomFailureSummaries))) + + # Add number of initial failing tests 'ift' scanned to summary + # data list + cdashReportData.summaryLineDataNumbersList.append( + "ift="+str(len(initialNonpassingTestsLOD))) + + for summary in randomFailureSummaries: + print(str(summary)) + summary.singleSummaryReporter(cdashReportData) + + summaryLine = CDQAR.getOverallCDashReportSummaryLine( + cdashReportData, cdashProjectName+" "+groupName, dateRangeStr) + print("\n"+summaryLine) + + # Finish HTML body paragraph + cdashReportData.htmlEmailBodyTop += "\n

" + + defaultPageStyle = CDQAR.getDefaultHtmlPageStyleStr() + + if writeEmailToFile: + print("\nWriting HTML to file: "+writeEmailToFile+" ...") + htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, + pageTitle=summaryLine, pageStyle=defaultPageStyle) + # print(htmlStr) + with open(writeEmailToFile, 'w') as outFile: + outFile.write(htmlStr) + + if sendEmailTo: + htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, + pageStyle=defaultPageStyle) + for emailAddress in sendEmailTo.split(','): + emailAddress = emailAddress.strip() + print("\nSending email to '"+emailAddress+"' ...") + msg=CDQAR.createHtmlMimeEmail( + sendEmailFrom, emailAddress, summaryLine, "", htmlStr) + CDQAR.sendMineEmail(msg) + + def getCmndLineArgs(self): + parser = argparse.ArgumentParser("Arguments for cdash_analyze_and_report_random_failures.py") + + parser.add_argument("--cdash-site-url", default="", required=True) + parser.add_argument("--cdash-project-name", default="", required=True) + parser.add_argument("--initial-nonpassing-test-filters", default="", required=True) + parser.add_argument("--group-name", default="", required=True) + parser.add_argument("--reference-date", default="yesterday") + parser.add_argument("--days-of-history", default=1, type=int) + parser.add_argument("--print-url-mode", choices=['none','initial','all'], default='none') + parser.add_argument("--write-email-to-file", default="") + parser.add_argument("--send-email-to", default="") + parser.add_argument("--send-email-from", default="random-failure-script@noreply.org") + + self.args = parser.parse_args() + + def getDateRangeTuple(self, referenceDateTime, dayTimeDelta): + beginDateTime = referenceDateTime - datetime.timedelta(days=(dayTimeDelta-1)) + beginDateTimeStr = CBTD.getDateStrFromDateTime(beginDateTime) + endDateTimeStr = CBTD.getDateStrFromDateTime(referenceDateTime) + return (beginDateTimeStr, endDateTimeStr) + + def getBuildIdFromTest(self, test): + return test['buildSummaryLink'].split("/")[-1] + + def downloadBuildSummaryOffCDash(self, + cdashBuildSummaryQueryUrl, buildSummaryCacheFile=None, + useCachedCDashData=False, alwaysUseCacheFileIfExists=False, + verbose='False' + ): + print("\n BUILD CACHE FILE "+buildSummaryCacheFile) + verbose = verbose == 'all' + buildSummaryJson = CDQAR.getAndCacheCDashQueryDataOrReadFromCache( + cdashBuildSummaryQueryUrl, buildSummaryCacheFile, useCachedCDashData, + alwaysUseCacheFileIfExists, verbose) + return buildSummaryJson + + +class RandomFailureSummary(object): + + def __init__(self, buildName, testName, testHistoryUrl, sha1Pair): + self.buildName = buildName + self.testName = testName + self.testHistoryUrl = testHistoryUrl + self.sha1Pair = sha1Pair + + def __str__(self): + myStr = "Test name: "+self.testName +\ + "\nBuild name: "+self.buildName +\ + "\nIdentical sha1 pairs: "+str(self.sha1Pair) +\ + "\nTest history browser URL: " +\ + "\n "+self.testHistoryUrl+"\n" + return myStr + + def singleSummaryReporter(self, cdashReportData): + cdashReportData.htmlEmailBodyTop += \ + "\n
Build name: "+ self.buildName +\ + "\n
Test name: "+ self.testName +\ + "\n
Test history URL: "+ self.testHistoryUrl +\ + "\n
Sha1 Pair : "+ str(self.sha1Pair) \ No newline at end of file diff --git a/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py b/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py index 317420080..83a29cc4b 100755 --- a/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py +++ b/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py @@ -18,11 +18,15 @@ def main(): class ExampleVersionInfoStrategy: - def getTopicTargetSha1s(buildData): - pass + def getTopicTargetSha1s(self, buildData): + pattern = r"Parent [12]:\n\s+(\w+)" + matchedList = regex.findall(pattern, buildData) - def checkTargetTopicRandomFailure(targetTopic, knownTargetTopics): - pass + if len(matchedList) != 2: return None + return tuple(matchedList) + + def checkTargetTopicRandomFailure(self, targetTopicPair, knownTargetTopicPairs): + return targetTopicPair in knownTargetTopicPairs class ExtractBuildNameStrategy: pass