From 44067c36940a3de4b72bef4f9b2be5ad10a793e8 Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 9 Feb 2024 12:06:19 -0700 Subject: [PATCH 1/9] Create module file CDashAnalyzeReportRandomFailures and example script (#600) Create driver class CDashAnalyzeReportRandomFailuresDriver inside module file CDashAnalyzeReportRandomFailures.py that will contain the main general functionality of the random test failure tool. The driver class accepts two strategy classes passed from the example script. These strategy classes ExampleVersionInfoStrategy and ExampleBuildNameStrategy contain the project specific implementation that is generically used inside of the driver class. --- .../CDashAnalyzeReportRandomFailures.py | 16 +++++++++ ...dash_analyze_and_report_random_failures.py | 33 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tribits/ci_support/CDashAnalyzeReportRandomFailures.py create mode 100755 tribits/ci_support/example_cdash_analyze_and_report_random_failures.py diff --git a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py new file mode 100644 index 000000000..c9346c2bb --- /dev/null +++ b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py @@ -0,0 +1,16 @@ + +import os +import argparse + +from FindGeneralScriptSupport import * +import CDashQueryAnalyzeReport as CDQAR + +class CDashAnalyzeReportRandomFailuresDriver: + + def __init__(self, versionInfoStrategy, extractBuildNameStrategy): + self.getTargetTopicSha1Strategy = versionInfoStrategy + self.getextractBuildNameStrategy = extractBuildNameStrategy + self.args = None + + def runDriver(self): + pass diff --git a/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py b/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py new file mode 100755 index 000000000..317420080 --- /dev/null +++ b/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import sys +import argparse +import re as regex + +import CDashAnalyzeReportRandomFailures as CDARRF + + +def main(): + + cdashAnalyzeAndReportRandomFailures = \ + CDARRF.CDashAnalyzeReportRandomFailuresDriver( + ExampleVersionInfoStrategy(), + ExtractBuildNameStrategy()) + + cdashAnalyzeAndReportRandomFailures.runDriver() + +class ExampleVersionInfoStrategy: + + def getTopicTargetSha1s(buildData): + pass + + def checkTargetTopicRandomFailure(targetTopic, knownTargetTopics): + pass + +class ExtractBuildNameStrategy: + pass + + + +if __name__ == '__main__': + sys.exit(main()) From 0f9fef374adc4e430829504da79171b7274f954d Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 9 Feb 2024 12:57:01 -0700 Subject: [PATCH 2/9] Copy cdash_analyze_and_report_random_failures.py main into module file (#600) This large commit is copying over the main() function and its associated helper functions into CDashAnalyzeReportRandomFailures.py inside the CDashAnalyzeReportRandomFailuresDriver class. This is part of the effort to refactor cdash_analyze_and_report_random_failures.py to be more generic. --- ...ze_and_report_random_failures_UnitTests.py | 2 +- .../CDashAnalyzeReportRandomFailures.py | 307 +++++++++++++++++- ...dash_analyze_and_report_random_failures.py | 12 +- 3 files changed, 313 insertions(+), 8 deletions(-) diff --git a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py index 1678a0979..a7d0ccfb9 100644 --- a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py +++ b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py @@ -152,7 +152,7 @@ def cdash_analyze_and_report_random_failures_run_case( cmnd = ( ciSupportDir - + "/cdash_analyze_and_report_random_failures.py" + + "/example_cdash_analyze_and_report_random_failures.py" + " --cdash-project-name='Project Name'" + " --group-name='Group Name'" + " --initial-nonpassing-test-filters='initial_nonpassing_test_filters'" diff --git a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py index c9346c2bb..52add5887 100644 --- a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py +++ b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py @@ -1,16 +1,317 @@ import os import argparse +import datetime from FindGeneralScriptSupport import * import CDashQueryAnalyzeReport as CDQAR +import cdash_build_testing_date as CBTD + class CDashAnalyzeReportRandomFailuresDriver: def __init__(self, versionInfoStrategy, extractBuildNameStrategy): - self.getTargetTopicSha1Strategy = versionInfoStrategy - self.getextractBuildNameStrategy = extractBuildNameStrategy + self.versionInfoStrategy = versionInfoStrategy + self.extractBuildNameStrategy = extractBuildNameStrategy self.args = None def runDriver(self): - pass + + self.getCmndLineArgs() + + cdashProjectTestingDayStartTime = "00:00" + # TODO: This should be moved outside in a project specific + # driver script or command line input + cdashSiteUrl = self.args.cdash_site_url + cdashProjectName = self.args.cdash_project_name + initialNonpassingTestFilters = self.args.initial_nonpassing_test_filters + date = self.args.reference_date + groupName = self.args.group_name + daysOfHistory = self.args.days_of_history + printUrlMode = self.args.print_url_mode + writeEmailToFile = self.args.write_email_to_file + sendEmailFrom = self.args.send_email_from + sendEmailTo = self.args.send_email_to + + randomFailureSummaries = [] + + # A.1) Set up date range and directories + + # Construct date range for queryTests filter string + referenceDateDT = CDQAR.convertInputDateArgToYYYYMMDD( + cdashProjectTestingDayStartTime, date) + dateRangeStart, dateRangeEnd = self.getDateRangeTuple(referenceDateDT, daysOfHistory) + dateUrlField = "begin="+dateRangeStart+"&end="+dateRangeEnd + dateRangeStr = dateRangeStart+" to "+dateRangeEnd + + testQueriesCacheDir = os.getcwd()+"/test_queries_cache" + testHistoryCacheDir = testQueriesCacheDir+"/test_history_cache" + createDirsFromPath(testQueriesCacheDir) + createDirsFromPath(testHistoryCacheDir) + + # Construct queryTest.php filter for a date range + initialNonpassingTestQueryFilters = \ + dateUrlField+"&"+initialNonpassingTestFilters + + # A.2) Create starting email body and html string aggregation var + + cdashReportData = CDQAR.CDashReportData() + + cdashReportData.htmlEmailBodyTop += \ + "

Random test failure scan results for "+cdashProjectName\ + +" from "+dateRangeStr+"

\n\n" + + + # B.1) Get all failing test result for past daysOfHistory + + # Beginning of scanned details and links paragraph + cdashReportData.htmlEmailBodyTop +="

\n" + + print("\nGetting list of initial nonpassing tests from CDash from "+dateRangeStr) + + initialNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( + cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) + initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( + cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) + + if printUrlMode == 'initial' or printUrlMode == 'all': + print("\nCDash nonpassing tests browser URL:\n\n"+\ + " "+initialNonpassingTestBrowserUrl+"\n") + print("\nCDash nonpassing tests query URL:\n\n"+\ + " "+initialNonpassingTestsQueryUrl+"\n") + + initialNonpassingTestsQueryCacheFile = testQueriesCacheDir +\ + "/initialCDashNonPassingTests_"+dateRangeStart+"_"+dateRangeEnd+".json" + + # List of dictionaries containing the cdash results in rows + initialNonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( + initialNonpassingTestsQueryUrl, initialNonpassingTestsQueryCacheFile,\ + alwaysUseCacheFileIfExists=True) + + cdashReportData.htmlEmailBodyTop += \ + "" +\ + "Nonpassing tests scanned on CDash=" +\ + str(len(initialNonpassingTestsLOD))+"
\n" + + # Ending of scanned details and links paragraph + # and start of scanning summaries and table + cdashReportData.htmlEmailBodyTop +="

\n\n

\n" + + # B.2) Get each nonpassing test's testing history + for nonpassingTest in initialNonpassingTestsLOD: + + # Remove unique jenkins run ID from build name + correctedBuildName = nonpassingTest['buildName'].rsplit('-', 1)[0] + # NOTE: This is project specific code. As Ross pointed out, buildName + # contains a lot of Trilinos specific prefix and suffix that should + # be processed by a Strategy class to be more project agnostic. + + buildNameMax = 80 + shortenedBuildName = correctedBuildName[:buildNameMax] + + print("\n Getting history from "+dateRangeStr+" for\n"+\ + " Test name: "+nonpassingTest['testname']+"\n"+\ + " Build name: "+correctedBuildName) + + groupNameNormUrl, = CDQAR.normalizeUrlStrings(groupName) + + testHistoryFilters = \ + "filtercount=3&showfilters=1&filtercombine=and"+\ + "&field1=testname&compare1=63&value1="+nonpassingTest['testname']+\ + "&field2=groupname&compare2=63&value2="+groupNameNormUrl+\ + "&field3=buildname&compare3=63&value3="+correctedBuildName + + testHistoryQueryFilters = dateUrlField+"&"+testHistoryFilters + + testHistoryCacheFile = testHistoryCacheDir+"/" +\ + nonpassingTest['testname']+"_"+shortenedBuildName+".json" + + print("\n Creating file to write test history:\n "+testHistoryCacheFile) + + testHistoryQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( + cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters) + testHistoryBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( + cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters) + + testHistoryLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( + testHistoryQueryUrl, testHistoryCacheFile, alwaysUseCacheFileIfExists=True) + + print("\n Size of test history: "+str(len(testHistoryLOD))) + + if printUrlMode == 'all': + print("\n CDash test history browser URL for "+nonpassingTest['testname']+" "+\ + correctedBuildName+":\n\n"+" "+testHistoryBrowserUrl) + print("\n CDash test history query URL for "+nonpassingTest['testname']+" "+\ + correctedBuildName+":\n\n"+" "+testHistoryQueryUrl) + + if len(testHistoryLOD) < 2: + print("\n Size of test history too small for any comparisons, skipping ...\n") + continue + + # B.3) Split full testing history to passed and nonpassed lists of dicts + passingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Passed"] + nonpassingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Failed"] + nonpassingSha1Pairs = set() + + print("\n Num of passing tests in test history: "+str(len(passingTestHistoryLOD))) + print("\n Num of nonpassing tests in test history: "+str(len(nonpassingTestHistoryLOD))) + + buildSummaryCacheDir = testQueriesCacheDir+"/build_summary_cache/" +\ + nonpassingTest['testname']+"_"+shortenedBuildName + createDirsFromPath(buildSummaryCacheDir) + # NOTE: There is an argument to be made that test histories should get their own directory + # instead of build summaries and that build summaries should live inside of there + + # C.1) Get all nonpassing tests' sha1s into a set + for test in nonpassingTestHistoryLOD: + + buildId = self.getBuildIdFromTest(test) + + buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId + buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId) + buildConfigOutput = self.downloadBuildSummaryOffCDash( + buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all', + alwaysUseCacheFileIfExists=True)['configure']['output'] + + nonpassingSha1Pairs.add( + self.versionInfoStrategy.getTopicTargetSha1s(buildConfigOutput)) + # nonpassingSha1Pairs.add(getTopicTargetSha1s(buildConfigOutput)) + + print("\n Test history failing sha1s: "+str(nonpassingSha1Pairs)) + + # C.2) Check if passing tests' sha1s exist in nonpassing sha1s set + for test in passingTestHistoryLOD: + + buildId = self.getBuildIdFromTest(test) + + buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId + buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId) + buildConfigOutput = self.downloadBuildSummaryOffCDash( + buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all', + alwaysUseCacheFileIfExists=True)['configure']['output'] + + passingSha1Pair = \ + self.versionInfoStrategy.getTopicTargetSha1s(buildConfigOutput) + + if self.versionInfoStrategy.checkTargetTopicRandomFailure(passingSha1Pair, nonpassingSha1Pairs): + print("\n Found passing sha1 pair, " + str(passingSha1Pair)+\ + " in set of nonpassing sha1 pairs: \n"+str(nonpassingSha1Pairs)) + + randomFailureSummaries.append( + RandomFailureSummary(test['buildName'], test['testname'], + testHistoryBrowserUrl, passingSha1Pair)) + + + print("\n*** CDash random failure analysis for " +\ + cdashProjectName+" "+groupName+" from " +dateRangeStr) + + print("Total number of initial failing tests: "+str(len(initialNonpassingTestsLOD))+"\n") + + print("Found random failing tests: "+str(len(randomFailureSummaries))+"\n") + + cdashReportData.htmlEmailBodyTop += \ + "Found random failing tests: "+str(len(randomFailureSummaries))+"
\n" + + if len(randomFailureSummaries) > 0: + cdashReportData.globalPass = False + + # Add number of random failing tests 'rft' found to summary data list + cdashReportData.summaryLineDataNumbersList.append( + "rft="+str(len(randomFailureSummaries))) + + # Add number of initial failing tests 'ift' scanned to summary + # data list + cdashReportData.summaryLineDataNumbersList.append( + "ift="+str(len(initialNonpassingTestsLOD))) + + for summary in randomFailureSummaries: + print(str(summary)) + summary.singleSummaryReporter(cdashReportData) + + summaryLine = CDQAR.getOverallCDashReportSummaryLine( + cdashReportData, cdashProjectName+" "+groupName, dateRangeStr) + print("\n"+summaryLine) + + # Finish HTML body paragraph + cdashReportData.htmlEmailBodyTop += "\n

" + + defaultPageStyle = CDQAR.getDefaultHtmlPageStyleStr() + + if writeEmailToFile: + print("\nWriting HTML to file: "+writeEmailToFile+" ...") + htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, + pageTitle=summaryLine, pageStyle=defaultPageStyle) + # print(htmlStr) + with open(writeEmailToFile, 'w') as outFile: + outFile.write(htmlStr) + + if sendEmailTo: + htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, + pageStyle=defaultPageStyle) + for emailAddress in sendEmailTo.split(','): + emailAddress = emailAddress.strip() + print("\nSending email to '"+emailAddress+"' ...") + msg=CDQAR.createHtmlMimeEmail( + sendEmailFrom, emailAddress, summaryLine, "", htmlStr) + CDQAR.sendMineEmail(msg) + + def getCmndLineArgs(self): + parser = argparse.ArgumentParser("Arguments for cdash_analyze_and_report_random_failures.py") + + parser.add_argument("--cdash-site-url", default="", required=True) + parser.add_argument("--cdash-project-name", default="", required=True) + parser.add_argument("--initial-nonpassing-test-filters", default="", required=True) + parser.add_argument("--group-name", default="", required=True) + parser.add_argument("--reference-date", default="yesterday") + parser.add_argument("--days-of-history", default=1, type=int) + parser.add_argument("--print-url-mode", choices=['none','initial','all'], default='none') + parser.add_argument("--write-email-to-file", default="") + parser.add_argument("--send-email-to", default="") + parser.add_argument("--send-email-from", default="random-failure-script@noreply.org") + + self.args = parser.parse_args() + + def getDateRangeTuple(self, referenceDateTime, dayTimeDelta): + beginDateTime = referenceDateTime - datetime.timedelta(days=(dayTimeDelta-1)) + beginDateTimeStr = CBTD.getDateStrFromDateTime(beginDateTime) + endDateTimeStr = CBTD.getDateStrFromDateTime(referenceDateTime) + return (beginDateTimeStr, endDateTimeStr) + + def getBuildIdFromTest(self, test): + return test['buildSummaryLink'].split("/")[-1] + + def downloadBuildSummaryOffCDash(self, + cdashBuildSummaryQueryUrl, buildSummaryCacheFile=None, + useCachedCDashData=False, alwaysUseCacheFileIfExists=False, + verbose='False' + ): + print("\n BUILD CACHE FILE "+buildSummaryCacheFile) + verbose = verbose == 'all' + buildSummaryJson = CDQAR.getAndCacheCDashQueryDataOrReadFromCache( + cdashBuildSummaryQueryUrl, buildSummaryCacheFile, useCachedCDashData, + alwaysUseCacheFileIfExists, verbose) + return buildSummaryJson + + +class RandomFailureSummary(object): + + def __init__(self, buildName, testName, testHistoryUrl, sha1Pair): + self.buildName = buildName + self.testName = testName + self.testHistoryUrl = testHistoryUrl + self.sha1Pair = sha1Pair + + def __str__(self): + myStr = "Test name: "+self.testName +\ + "\nBuild name: "+self.buildName +\ + "\nIdentical sha1 pairs: "+str(self.sha1Pair) +\ + "\nTest history browser URL: " +\ + "\n "+self.testHistoryUrl+"\n" + return myStr + + def singleSummaryReporter(self, cdashReportData): + cdashReportData.htmlEmailBodyTop += \ + "\n
Build name: "+ self.buildName +\ + "\n
Test name: "+ self.testName +\ + "\n
Test history URL: "+ self.testHistoryUrl +\ + "\n
Sha1 Pair : "+ str(self.sha1Pair) \ No newline at end of file diff --git a/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py b/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py index 317420080..83a29cc4b 100755 --- a/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py +++ b/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py @@ -18,11 +18,15 @@ def main(): class ExampleVersionInfoStrategy: - def getTopicTargetSha1s(buildData): - pass + def getTopicTargetSha1s(self, buildData): + pattern = r"Parent [12]:\n\s+(\w+)" + matchedList = regex.findall(pattern, buildData) - def checkTargetTopicRandomFailure(targetTopic, knownTargetTopics): - pass + if len(matchedList) != 2: return None + return tuple(matchedList) + + def checkTargetTopicRandomFailure(self, targetTopicPair, knownTargetTopicPairs): + return targetTopicPair in knownTargetTopicPairs class ExtractBuildNameStrategy: pass From 4214e32a829249453333dd084c7bcf32c6f0f0c7 Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 9 Feb 2024 13:18:47 -0700 Subject: [PATCH 3/9] Renamed ExampleVersionInfoStrategy member function to be consistent (#600) There were mixed use cases of 'targetTopic' or 'topicTarget', this renames all cases to use 'targetTopic' approach. --- tribits/ci_support/CDashAnalyzeReportRandomFailures.py | 5 ++--- .../example_cdash_analyze_and_report_random_failures.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py index 52add5887..e45b8b819 100644 --- a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py +++ b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py @@ -174,8 +174,7 @@ def runDriver(self): alwaysUseCacheFileIfExists=True)['configure']['output'] nonpassingSha1Pairs.add( - self.versionInfoStrategy.getTopicTargetSha1s(buildConfigOutput)) - # nonpassingSha1Pairs.add(getTopicTargetSha1s(buildConfigOutput)) + self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput)) print("\n Test history failing sha1s: "+str(nonpassingSha1Pairs)) @@ -191,7 +190,7 @@ def runDriver(self): alwaysUseCacheFileIfExists=True)['configure']['output'] passingSha1Pair = \ - self.versionInfoStrategy.getTopicTargetSha1s(buildConfigOutput) + self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput) if self.versionInfoStrategy.checkTargetTopicRandomFailure(passingSha1Pair, nonpassingSha1Pairs): print("\n Found passing sha1 pair, " + str(passingSha1Pair)+\ diff --git a/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py b/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py index 83a29cc4b..01b19385e 100755 --- a/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py +++ b/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py @@ -18,7 +18,7 @@ def main(): class ExampleVersionInfoStrategy: - def getTopicTargetSha1s(self, buildData): + def getTargetTopicSha1s(self, buildData): pattern = r"Parent [12]:\n\s+(\w+)" matchedList = regex.findall(pattern, buildData) From ea196b105afd9efd275aa9a1e77b5e221e24a62f Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 9 Feb 2024 14:14:22 -0700 Subject: [PATCH 4/9] Moved example_cdash_analyze_and_report_random_failures.py (#600) Moved example_cdash_analyze_and_report_random_failures.py to test/ci_support --- .../cdash_analyze_and_report_random_failures_UnitTests.py | 2 +- .../example_cdash_analyze_and_report_random_failures.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) rename {tribits => test}/ci_support/example_cdash_analyze_and_report_random_failures.py (96%) diff --git a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py index a7d0ccfb9..b2f8216f6 100644 --- a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py +++ b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py @@ -151,7 +151,7 @@ def cdash_analyze_and_report_random_failures_run_case( htmlFileAbsPath = os.getcwd()+"/"+htmlFileName cmnd = ( - ciSupportDir + testCiSupportDir + "/example_cdash_analyze_and_report_random_failures.py" + " --cdash-project-name='Project Name'" + " --group-name='Group Name'" diff --git a/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py b/test/ci_support/example_cdash_analyze_and_report_random_failures.py similarity index 96% rename from tribits/ci_support/example_cdash_analyze_and_report_random_failures.py rename to test/ci_support/example_cdash_analyze_and_report_random_failures.py index 01b19385e..6ac94e6f3 100755 --- a/tribits/ci_support/example_cdash_analyze_and_report_random_failures.py +++ b/test/ci_support/example_cdash_analyze_and_report_random_failures.py @@ -4,6 +4,7 @@ import argparse import re as regex +from FindCISupportDir import * import CDashAnalyzeReportRandomFailures as CDARRF From dbe94f495267e81a7f98107468816c08c580b639 Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 9 Feb 2024 14:44:45 -0700 Subject: [PATCH 5/9] Implemented Trilinos specific driver (#600) Trilinos specific driver `trilinos_cdash_analyze_and_report_random_failures.py` based on `example_cdash_analyze_and_report_random_failures.py` that contains the Trilinos specific implementations of `VersionInfoStrategy` and `ExtractBuildNameStrategy`. --- ...dash_analyze_and_report_random_failures.py | 40 ++++++++++++++ ...ze_and_report_random_failures_UnitTests.py | 53 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100755 test/ci_support/trilinos_cdash_analyze_and_report_random_failures.py create mode 100644 test/ci_support/trilinos_cdash_analyze_and_report_random_failures_UnitTests.py diff --git a/test/ci_support/trilinos_cdash_analyze_and_report_random_failures.py b/test/ci_support/trilinos_cdash_analyze_and_report_random_failures.py new file mode 100755 index 000000000..6d27a35a5 --- /dev/null +++ b/test/ci_support/trilinos_cdash_analyze_and_report_random_failures.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +import sys +import argparse +import re as regex + +import CDashAnalyzeReportRandomFailures as CDARRF + + +def main(): + + cdashAnalyzeAndReportRandomFailures = \ + CDARRF.CDashAnalyzeReportRandomFailuresDriver( + TrilinosVersionInfoStrategy(), + ExtractBuildNameStrategy()) + + cdashAnalyzeAndReportRandomFailures.runDriver() + +class TrilinosVersionInfoStrategy: + + def getTopicTargetSha1s(self, buildData): + pattern = r"Parent [12]:\n\s+(\w+)" + matchedList = regex.findall(pattern, buildData) + + if len(matchedList) != 2: return None + return tuple(matchedList) + + def checkTargetTopicRandomFailure(self, targetTopicPair, knownTargetTopicPairs): + return targetTopicPair in knownTargetTopicPairs + +class TrilinosExtractBuildNameStrategy: + + def getCoreBuildName(self, fullBuildName): + coreBuildName = fullBuildName.rsplit('-',1)[0] + return coreBuildName + + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/ci_support/trilinos_cdash_analyze_and_report_random_failures_UnitTests.py b/test/ci_support/trilinos_cdash_analyze_and_report_random_failures_UnitTests.py new file mode 100644 index 000000000..b207a16ab --- /dev/null +++ b/test/ci_support/trilinos_cdash_analyze_and_report_random_failures_UnitTests.py @@ -0,0 +1,53 @@ + +import unittest + +from FindCISupportDir import * +from trilinos_cdash_analyze_and_report_random_failures import * + + +class TestTrilinosVersionInfoStrategy(unittest.TestCase): + + def setUp(self): + self.strategy = TrilinosVersionInfoStrategy() + + def test_getTargetTopicSha1s_with_valid_versionData(self): + versionData = "Parent 1:\n target\nParent 2:\n topic" + + expected = ('target','topic') + result = self.strategy.getTopicTargetSha1s(versionData) + + self.assertEqual(result, expected) + + def test_checkTargetTopicRandomFailure_true(self): + targetTopicPair = ('target', 'topic') + knownTargetTopicPairs = {('target','topic')} + + result = self.strategy.checkTargetTopicRandomFailure(targetTopicPair, knownTargetTopicPairs) + + self.assertTrue(result) + + def test_checkTargetTopicRandomFailure_false(self): + targetTopicPair = ('some','other') + knownTargetTopicPairs = {('target','topic')} + + result = self.strategy.checkTargetTopicRandomFailure(targetTopicPair, knownTargetTopicPairs) + + self.assertFalse(result) + + +class TrilinosExtractBuildName(unittest.TestCase): + + def setUp(self): + self.strategy = TrilinosExtractBuildNameStrategy() + + def test_getCoreBuildName(self): + fullBuildName = "PR-number-test-core_build-name-number" + + expected = "PR-number-test-core_build-name" + result = self.strategy.getCoreBuildName(fullBuildName) + + self.assertEqual(result, expected) + + +if __name__ == '__main__': + unittest.main() From ac8736f68fa831ba157f526c92a80537cdf30fe0 Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 9 Feb 2024 14:55:34 -0700 Subject: [PATCH 6/9] Fix type of ExampleExtractBuildNameStrategy class (#600) Example class did not include the 'Example' prefix. --- .../example_cdash_analyze_and_report_random_failures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/ci_support/example_cdash_analyze_and_report_random_failures.py b/test/ci_support/example_cdash_analyze_and_report_random_failures.py index 6ac94e6f3..72da4853e 100755 --- a/test/ci_support/example_cdash_analyze_and_report_random_failures.py +++ b/test/ci_support/example_cdash_analyze_and_report_random_failures.py @@ -13,7 +13,7 @@ def main(): cdashAnalyzeAndReportRandomFailures = \ CDARRF.CDashAnalyzeReportRandomFailuresDriver( ExampleVersionInfoStrategy(), - ExtractBuildNameStrategy()) + ExampleExtractBuildNameStrategy()) cdashAnalyzeAndReportRandomFailures.runDriver() @@ -29,7 +29,7 @@ def getTargetTopicSha1s(self, buildData): def checkTargetTopicRandomFailure(self, targetTopicPair, knownTargetTopicPairs): return targetTopicPair in knownTargetTopicPairs -class ExtractBuildNameStrategy: +class ExampleExtractBuildNameStrategy: pass From ba69e6f4ee496150bb79b1897364ce4f221589e2 Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 9 Feb 2024 15:00:30 -0700 Subject: [PATCH 7/9] Implemented Trilinos versions for Example driver (#600) This is for testing the CDashAnalyzeReportRandomFailures.py runDriver(). --- .../example_cdash_analyze_and_report_random_failures.py | 7 ++++++- tribits/ci_support/CDashAnalyzeReportRandomFailures.py | 5 +---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test/ci_support/example_cdash_analyze_and_report_random_failures.py b/test/ci_support/example_cdash_analyze_and_report_random_failures.py index 72da4853e..a6c9e7cba 100755 --- a/test/ci_support/example_cdash_analyze_and_report_random_failures.py +++ b/test/ci_support/example_cdash_analyze_and_report_random_failures.py @@ -30,7 +30,12 @@ def checkTargetTopicRandomFailure(self, targetTopicPair, knownTargetTopicPairs): return targetTopicPair in knownTargetTopicPairs class ExampleExtractBuildNameStrategy: - pass + + def getCoreBuildName(self, fullBuildName): + coreBuildName = fullBuildName.rsplit('-',1)[0] + return coreBuildName + + diff --git a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py index e45b8b819..ebc0d9364 100644 --- a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py +++ b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py @@ -101,10 +101,7 @@ def runDriver(self): for nonpassingTest in initialNonpassingTestsLOD: # Remove unique jenkins run ID from build name - correctedBuildName = nonpassingTest['buildName'].rsplit('-', 1)[0] - # NOTE: This is project specific code. As Ross pointed out, buildName - # contains a lot of Trilinos specific prefix and suffix that should - # be processed by a Strategy class to be more project agnostic. + correctedBuildName = self.extractBuildNameStrategy.getCoreBuildName(nonpassingTest['buildName']) buildNameMax = 80 shortenedBuildName = correctedBuildName[:buildNameMax] From 63c0592a18f94ff9b49118ee56786a11552aeb0d Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 9 Feb 2024 17:30:38 -0700 Subject: [PATCH 8/9] Cleaned up spacing (#600) Adjusted spacing between classes and added newline character at the end of file. --- .../example_cdash_analyze_and_report_random_failures.py | 5 ++--- tribits/ci_support/CDashAnalyzeReportRandomFailures.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/ci_support/example_cdash_analyze_and_report_random_failures.py b/test/ci_support/example_cdash_analyze_and_report_random_failures.py index a6c9e7cba..1d8b7c398 100755 --- a/test/ci_support/example_cdash_analyze_and_report_random_failures.py +++ b/test/ci_support/example_cdash_analyze_and_report_random_failures.py @@ -17,6 +17,7 @@ def main(): cdashAnalyzeAndReportRandomFailures.runDriver() + class ExampleVersionInfoStrategy: def getTargetTopicSha1s(self, buildData): @@ -29,6 +30,7 @@ def getTargetTopicSha1s(self, buildData): def checkTargetTopicRandomFailure(self, targetTopicPair, knownTargetTopicPairs): return targetTopicPair in knownTargetTopicPairs + class ExampleExtractBuildNameStrategy: def getCoreBuildName(self, fullBuildName): @@ -36,8 +38,5 @@ def getCoreBuildName(self, fullBuildName): return coreBuildName - - - if __name__ == '__main__': sys.exit(main()) diff --git a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py index ebc0d9364..8a9f39567 100644 --- a/tribits/ci_support/CDashAnalyzeReportRandomFailures.py +++ b/tribits/ci_support/CDashAnalyzeReportRandomFailures.py @@ -310,4 +310,5 @@ def singleSummaryReporter(self, cdashReportData): "\n
Build name: "+ self.buildName +\ "\n
Test name: "+ self.testName +\ "\n
Test history URL: "+ self.testHistoryUrl +\ - "\n
Sha1 Pair : "+ str(self.sha1Pair) \ No newline at end of file + "\n
Sha1 Pair : "+ str(self.sha1Pair) + From 913835832fba8cd48d97338eeea958a716dabbdb Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Fri, 9 Feb 2024 17:35:56 -0700 Subject: [PATCH 9/9] Revert "Implemented Trilinos specific driver (#600)" This reverts commit dbe94f495267e81a7f98107468816c08c580b639. Reverting this commit as this specific driver implementation shouldn't be existing inside of TriBITS. Rather it should be added to the Trilinos repo after snapshotting TriBITS in. --- ...dash_analyze_and_report_random_failures.py | 40 -------------- ...ze_and_report_random_failures_UnitTests.py | 53 ------------------- 2 files changed, 93 deletions(-) delete mode 100755 test/ci_support/trilinos_cdash_analyze_and_report_random_failures.py delete mode 100644 test/ci_support/trilinos_cdash_analyze_and_report_random_failures_UnitTests.py diff --git a/test/ci_support/trilinos_cdash_analyze_and_report_random_failures.py b/test/ci_support/trilinos_cdash_analyze_and_report_random_failures.py deleted file mode 100755 index 6d27a35a5..000000000 --- a/test/ci_support/trilinos_cdash_analyze_and_report_random_failures.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -import sys -import argparse -import re as regex - -import CDashAnalyzeReportRandomFailures as CDARRF - - -def main(): - - cdashAnalyzeAndReportRandomFailures = \ - CDARRF.CDashAnalyzeReportRandomFailuresDriver( - TrilinosVersionInfoStrategy(), - ExtractBuildNameStrategy()) - - cdashAnalyzeAndReportRandomFailures.runDriver() - -class TrilinosVersionInfoStrategy: - - def getTopicTargetSha1s(self, buildData): - pattern = r"Parent [12]:\n\s+(\w+)" - matchedList = regex.findall(pattern, buildData) - - if len(matchedList) != 2: return None - return tuple(matchedList) - - def checkTargetTopicRandomFailure(self, targetTopicPair, knownTargetTopicPairs): - return targetTopicPair in knownTargetTopicPairs - -class TrilinosExtractBuildNameStrategy: - - def getCoreBuildName(self, fullBuildName): - coreBuildName = fullBuildName.rsplit('-',1)[0] - return coreBuildName - - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/test/ci_support/trilinos_cdash_analyze_and_report_random_failures_UnitTests.py b/test/ci_support/trilinos_cdash_analyze_and_report_random_failures_UnitTests.py deleted file mode 100644 index b207a16ab..000000000 --- a/test/ci_support/trilinos_cdash_analyze_and_report_random_failures_UnitTests.py +++ /dev/null @@ -1,53 +0,0 @@ - -import unittest - -from FindCISupportDir import * -from trilinos_cdash_analyze_and_report_random_failures import * - - -class TestTrilinosVersionInfoStrategy(unittest.TestCase): - - def setUp(self): - self.strategy = TrilinosVersionInfoStrategy() - - def test_getTargetTopicSha1s_with_valid_versionData(self): - versionData = "Parent 1:\n target\nParent 2:\n topic" - - expected = ('target','topic') - result = self.strategy.getTopicTargetSha1s(versionData) - - self.assertEqual(result, expected) - - def test_checkTargetTopicRandomFailure_true(self): - targetTopicPair = ('target', 'topic') - knownTargetTopicPairs = {('target','topic')} - - result = self.strategy.checkTargetTopicRandomFailure(targetTopicPair, knownTargetTopicPairs) - - self.assertTrue(result) - - def test_checkTargetTopicRandomFailure_false(self): - targetTopicPair = ('some','other') - knownTargetTopicPairs = {('target','topic')} - - result = self.strategy.checkTargetTopicRandomFailure(targetTopicPair, knownTargetTopicPairs) - - self.assertFalse(result) - - -class TrilinosExtractBuildName(unittest.TestCase): - - def setUp(self): - self.strategy = TrilinosExtractBuildNameStrategy() - - def test_getCoreBuildName(self): - fullBuildName = "PR-number-test-core_build-name-number" - - expected = "PR-number-test-core_build-name" - result = self.strategy.getCoreBuildName(fullBuildName) - - self.assertEqual(result, expected) - - -if __name__ == '__main__': - unittest.main()