diff --git a/cmake/tribits/ci_support/CDashAnalyzeReportRandomFailures.py b/cmake/tribits/ci_support/CDashAnalyzeReportRandomFailures.py new file mode 100644 index 000000000000..4973bec16ed4 --- /dev/null +++ b/cmake/tribits/ci_support/CDashAnalyzeReportRandomFailures.py @@ -0,0 +1,339 @@ + +import os +import argparse +import datetime + +from FindGeneralScriptSupport import * +import CDashQueryAnalyzeReport as CDQAR +import cdash_build_testing_date as CBTD + + +class CDashAnalyzeReportRandomFailuresDriver: + + def __init__(self, versionInfoStrategy, extractBuildNameStrategy, + usageHelp=""): + self.versionInfoStrategy = versionInfoStrategy + self.extractBuildNameStrategy = extractBuildNameStrategy + self.args = None + self.usageHelp = usageHelp + + def runDriver(self): + + self.getCmndLineArgs() + + cdashProjectTestingDayStartTime = self.args.cdash_testing_day_start_time + cdashSiteUrl = self.args.cdash_site_url + cdashProjectName = self.args.cdash_project_name + initialNonpassingTestFilters = self.args.initial_nonpassing_test_filters + date = self.args.reference_date + groupName = self.args.group_name + daysOfHistory = self.args.days_of_history + printUrlMode = self.args.print_url_mode + writeEmailToFile = self.args.write_email_to_file + sendEmailFrom = self.args.send_email_from + sendEmailTo = self.args.send_email_to + + randomFailureSummaries = [] + + # A.1) Set up date range and directories + + # Construct date range for queryTests filter string + referenceDateDT = CDQAR.convertInputDateArgToYYYYMMDD( + cdashProjectTestingDayStartTime, date) + dateRangeStart, dateRangeEnd = self.getDateRangeTuple(referenceDateDT, daysOfHistory) + dateUrlField = "begin="+dateRangeStart+"&end="+dateRangeEnd + dateRangeStr = dateRangeStart+" to "+dateRangeEnd + + testQueriesCacheDir = os.getcwd()+"/test_queries_cache" + testHistoryCacheDir = testQueriesCacheDir+"/test_history_cache" + createDirsFromPath(testQueriesCacheDir) + createDirsFromPath(testHistoryCacheDir) + + # Construct queryTest.php filter for a date range + initialNonpassingTestQueryFilters = \ + dateUrlField+"&"+initialNonpassingTestFilters + + # A.2) Create starting email body and html string aggregation var + + cdashReportData = CDQAR.CDashReportData() + + cdashReportData.htmlEmailBodyTop += \ + "

Random test failure scan results for "+cdashProjectName\ + +" from "+dateRangeStr+"

\n\n" + + + # B.1) Get all failing test result for past daysOfHistory + + # Beginning of scanned details and links paragraph + cdashReportData.htmlEmailBodyTop +="

\n" + + print("\nGetting list of initial nonpassing tests from CDash from "+dateRangeStr) + + initialNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( + cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) + initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( + cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) + + if printUrlMode == 'initial' or printUrlMode == 'all': + print("\nCDash nonpassing tests browser URL:\n\n"+\ + " "+initialNonpassingTestBrowserUrl+"\n") + print("\nCDash nonpassing tests query URL:\n\n"+\ + " "+initialNonpassingTestsQueryUrl+"\n") + + initialNonpassingTestsQueryCacheFile = testQueriesCacheDir +\ + "/initialCDashNonPassingTests_"+dateRangeStart+"_"+dateRangeEnd+".json" + + # List of dictionaries containing the cdash results in rows + initialNonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( + initialNonpassingTestsQueryUrl, initialNonpassingTestsQueryCacheFile,\ + alwaysUseCacheFileIfExists=True) + + cdashReportData.htmlEmailBodyTop += \ + "" +\ + "Nonpassing tests scanned on CDash=" +\ + str(len(initialNonpassingTestsLOD))+"
\n" + + # Ending of scanned details and links paragraph + # and start of scanning summaries and table + cdashReportData.htmlEmailBodyTop +="

\n\n

\n" + + # B.2) Get each nonpassing test's testing history + for nonpassingTest in initialNonpassingTestsLOD: + + # Remove unique jenkins run ID from build name + correctedBuildName = self.extractBuildNameStrategy.getCoreBuildName(nonpassingTest['buildName']) + + testNameBuildName = nonpassingTest['testname']+"_"+nonpassingTest['buildName'] + testNameBuildName = CDQAR.getCompressedFileNameIfTooLong(testNameBuildName) + + print("\n Getting history from "+dateRangeStr+" for\n"+\ + " Test name: "+nonpassingTest['testname']+"\n"+\ + " Build name: "+correctedBuildName) + + groupNameNormUrl, = CDQAR.normalizeUrlStrings(groupName) + + testHistoryFilters = \ + "filtercount=3&showfilters=1&filtercombine=and"+\ + "&field1=testname&compare1=63&value1="+nonpassingTest['testname']+\ + "&field2=groupname&compare2=63&value2="+groupNameNormUrl+\ + "&field3=buildname&compare3=63&value3="+correctedBuildName + + testHistoryQueryFilters = dateUrlField+"&"+testHistoryFilters + + testHistoryCacheFile = testHistoryCacheDir+"/"+\ + CDQAR.getCompressedFileNameIfTooLong(testNameBuildName+".json", ext=".json") + + print("\n Creating file to write test history:\n "+testHistoryCacheFile) + + testHistoryQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( + cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters) + testHistoryBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( + cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters) + + testHistoryLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( + testHistoryQueryUrl, testHistoryCacheFile, alwaysUseCacheFileIfExists=True) + + print("\n Size of test history: "+str(len(testHistoryLOD))) + + if printUrlMode == 'all': + print("\n CDash test history browser URL for "+nonpassingTest['testname']+" "+\ + correctedBuildName+":\n\n"+" "+testHistoryBrowserUrl) + print("\n CDash test history query URL for "+nonpassingTest['testname']+" "+\ + correctedBuildName+":\n\n"+" "+testHistoryQueryUrl) + + if len(testHistoryLOD) < 2: + print("\n Size of test history too small for any comparisons, skipping ...\n") + continue + + # B.3) Split full testing history to passed and nonpassed lists of dicts + passingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Passed"] + nonpassingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Failed"] + nonpassingSha1Pairs = set() + + print("\n Num of passing tests in test history: "+str(len(passingTestHistoryLOD))) + print("\n Num of nonpassing tests in test history: "+str(len(nonpassingTestHistoryLOD))) + + buildSummaryCacheDir = testQueriesCacheDir+"/build_summary_cache/"+testNameBuildName + createDirsFromPath(buildSummaryCacheDir) + # NOTE: There is an argument to be made that test histories should get their own directory + # instead of build summaries and that build summaries should live inside of there + + # C.1) Get all nonpassing tests' sha1s into a set + for test in nonpassingTestHistoryLOD: + + buildId = self.getBuildIdFromTest(test) + + buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId + buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId) + buildConfigOutput = self.downloadBuildSummaryOffCDash( + buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all', + alwaysUseCacheFileIfExists=True)['configure']['output'] + + nonpassingSha1Pairs.add( + self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput)) + + print("\n Test history failing sha1s: "+str(nonpassingSha1Pairs)) + + # C.2) Check if passing tests' sha1s exist in nonpassing sha1s set + for test in passingTestHistoryLOD: + + buildId = self.getBuildIdFromTest(test) + + buildSummaryCacheFile = buildSummaryCacheDir+"/"+buildId + buildSummaryQueryUrl = CDQAR.getCDashBuildSummaryQueryUrl(cdashSiteUrl, buildId) + buildConfigOutput = self.downloadBuildSummaryOffCDash( + buildSummaryQueryUrl, buildSummaryCacheFile, verbose=printUrlMode =='all', + alwaysUseCacheFileIfExists=True)['configure']['output'] + + passingSha1Pair = \ + self.versionInfoStrategy.getTargetTopicSha1s(buildConfigOutput) + + if self.versionInfoStrategy.checkTargetTopicRandomFailure(passingSha1Pair, nonpassingSha1Pairs): + print("\n Found passing sha1 pair, " + str(passingSha1Pair)+\ + " in set of nonpassing sha1 pairs: \n"+str(nonpassingSha1Pairs)) + + randomFailureSummaries.append( + RandomFailureSummary(test['buildName'], test['testname'], + testHistoryBrowserUrl, passingSha1Pair)) + + + print("\n*** CDash random failure analysis for " +\ + cdashProjectName+" "+groupName+" from " +dateRangeStr) + + print("Total number of initial failing tests: "+str(len(initialNonpassingTestsLOD))+"\n") + + print("Found random failing tests: "+str(len(randomFailureSummaries))+"\n") + + cdashReportData.htmlEmailBodyTop += \ + "Found random failing tests: "+str(len(randomFailureSummaries))+"
\n" + + if len(randomFailureSummaries) > 0: + cdashReportData.globalPass = False + + # Add number of random failing tests 'rft' found to summary data list + cdashReportData.summaryLineDataNumbersList.append( + "rft="+str(len(randomFailureSummaries))) + + # Add number of initial failing tests 'ift' scanned to summary + # data list + cdashReportData.summaryLineDataNumbersList.append( + "ift="+str(len(initialNonpassingTestsLOD))) + + for summary in randomFailureSummaries: + print(str(summary)) + summary.singleSummaryReporter(cdashReportData) + + summaryLine = CDQAR.getOverallCDashReportSummaryLine( + cdashReportData, cdashProjectName+" "+groupName, dateRangeStr) + print("\n"+summaryLine) + + # Finish HTML body paragraph + cdashReportData.htmlEmailBodyTop += "\n

" + + defaultPageStyle = CDQAR.getDefaultHtmlPageStyleStr() + + if writeEmailToFile: + print("\nWriting HTML to file: "+writeEmailToFile+" ...") + htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, + pageTitle=summaryLine, pageStyle=defaultPageStyle) + # print(htmlStr) + with open(writeEmailToFile, 'w') as outFile: + outFile.write(htmlStr) + + if sendEmailTo: + htmlStr = CDQAR.getFullCDashHtmlReportPageStr(cdashReportData, + pageStyle=defaultPageStyle) + for emailAddress in sendEmailTo.split(','): + emailAddress = emailAddress.strip() + print("\nSending email to '"+emailAddress+"' ...") + msg=CDQAR.createHtmlMimeEmail( + sendEmailFrom, emailAddress, summaryLine, "", htmlStr) + CDQAR.sendMineEmail(msg) + + def getCmndLineArgs(self): + parser = argparse.ArgumentParser(description=self.usageHelp) + + parser.add_argument("--cdash-site-url", default="", required=True, + help="Base CDash site (e.g. 'https://testing.sandia.gov/cdash')."+\ + " [REQUIRED]") + parser.add_argument("--cdash-project-name", default="", required=True, + help="CDash project name (e.g. TriBITS)."+\ + " [REQUIRED]") + parser.add_argument("--initial-nonpassing-test-filters", default="", required=True, + help="Partial URL fragment containing CDash filters for the initial set"+\ + " of nonpassing tests (e.g. 'filtercount=2&showfilters=1&filtercombine=and&field1=status&compare1=63&"+\ + "value1=Failed&field2=groupname&compare2=63&value2=Pull%%20Request')."+\ + " [REQUIRED]") + parser.add_argument("--group-name", default="", required=True, + help="Name of the build group index for filtering a set of test (e.g. 'Pull Request')."+\ + " [REQUIRED]") + parser.add_argument("--cdash-testing-day-start-time", default="00:00", + help="CDash project's testing day start time in UTC format ':'."+\ + " [default='00:00']") + parser.add_argument("--reference-date", default="yesterday", + help="Starting reference date for querying failing tests as or"+\ + " special values 'today' or 'yesterday.'"+\ + " [default='yesterday']") + parser.add_argument("--days-of-history", default=1, type=int, + help="Number of days of testing history to query starting from the reference and going"+\ + " backwards in time."+\ + " [default=1]") + parser.add_argument("--print-url-mode", choices=['none','initial','all'], default='none', + help="Mode of url printing."+\ + " [default=none]") + parser.add_argument("--write-email-to-file", default="", + help="Name of file to write built email HTML to."+\ + " [default='']") + parser.add_argument("--send-email-to", default="", + help="Target email address to send script summary results to."+\ + " [default='']") + parser.add_argument("--send-email-from", default="random-failure-script@noreply.org", + help="Addressed sender of the script summary results email."+\ + " [default='random-failure-script@noreply.org']") + + self.args = parser.parse_args() + + def getDateRangeTuple(self, referenceDateTime, dayTimeDelta): + beginDateTime = referenceDateTime - datetime.timedelta(days=(dayTimeDelta-1)) + beginDateTimeStr = CBTD.getDateStrFromDateTime(beginDateTime) + endDateTimeStr = CBTD.getDateStrFromDateTime(referenceDateTime) + return (beginDateTimeStr, endDateTimeStr) + + def getBuildIdFromTest(self, test): + return test['buildSummaryLink'].split("/")[-1] + + def downloadBuildSummaryOffCDash(self, + cdashBuildSummaryQueryUrl, buildSummaryCacheFile=None, + useCachedCDashData=False, alwaysUseCacheFileIfExists=False, + verbose='False' + ): + verbose = verbose == 'all' + buildSummaryJson = CDQAR.getAndCacheCDashQueryDataOrReadFromCache( + cdashBuildSummaryQueryUrl, buildSummaryCacheFile, useCachedCDashData, + alwaysUseCacheFileIfExists, verbose) + return buildSummaryJson + + +class RandomFailureSummary(object): + + def __init__(self, buildName, testName, testHistoryUrl, sha1Pair): + self.buildName = buildName + self.testName = testName + self.testHistoryUrl = testHistoryUrl + self.sha1Pair = sha1Pair + + def __str__(self): + myStr = "Test name: "+self.testName +\ + "\nBuild name: "+self.buildName +\ + "\nIdentical sha1 pairs: "+str(self.sha1Pair) +\ + "\nTest history browser URL: " +\ + "\n "+self.testHistoryUrl+"\n" + return myStr + + def singleSummaryReporter(self, cdashReportData): + cdashReportData.htmlEmailBodyTop += \ + "\n
Build name: "+ self.buildName +\ + "\n
Test name: "+ self.testName +\ + "\n
Test history URL: "+ self.testHistoryUrl +\ + "\n
Sha1 Pair : "+ str(self.sha1Pair) + diff --git a/cmake/tribits/ci_support/CDashQueryAnalyzeReport.py b/cmake/tribits/ci_support/CDashQueryAnalyzeReport.py index d8bc310cc676..804b8b97a5e7 100644 --- a/cmake/tribits/ci_support/CDashQueryAnalyzeReport.py +++ b/cmake/tribits/ci_support/CDashQueryAnalyzeReport.py @@ -769,6 +769,16 @@ def getCDashQueryTestsBrowserUrl(cdashUrl, projectName, date, filterFields): return cdashUrl+"/queryTests.php?project="+projectName+dateArg+"&"+filterFields +# Construct full cdash/api/v1/buildSummary.php query URL given the buildId +def getCDashBuildSummaryQueryUrl(cdashUrl, buildId): + return cdashUrl+"/api/v1/buildSummary.php?buildid="+buildId + + +# Construct full cdash/build browser URL given the buildId +def getCDashBuildSummaryBrowserUrl(cdashUrl, buildId): + return cdashUrl+"/build/"+buildId + + # Copy a key/value pair from one dict to another if it eixsts def copyKeyDictIfExists(sourceDict_in, keyName_in, dict_inout): value = sourceDict_in.get(keyName_in, None) diff --git a/cmake/tribits/core/package_arch/TribitsPackageDefineDependencies.cmake b/cmake/tribits/core/package_arch/TribitsPackageDefineDependencies.cmake index a5584fb85a08..5f916c10e931 100644 --- a/cmake/tribits/core/package_arch/TribitsPackageDefineDependencies.cmake +++ b/cmake/tribits/core/package_arch/TribitsPackageDefineDependencies.cmake @@ -128,9 +128,9 @@ include(TribitsGeneralMacros) # additional testing of some type. (Add these to # ``TEST_OPTIONAL_PACKAGES`` instead.) # -# NOTE: The ``XXX_TPLS`` arguments/lists are **deprecated**. At the package -# level, there is no distinction between upstream internal and external -# packages (i.e. TPLs) so all upstream package dependencies can (and should) +# NOTE: The above ``XXX_TPLS`` arguments/lists are **deprecated**. At the +# package level, there is no distinction between upstream internal and +# external packages/TPLs, so all upstream package dependencies can (and should) # be listed in the ``XXX_PACKAGES`` arguments/lists. (There is no change in # behavior listing upstream packages in ``XXX_PACKAGES`` or the ``XXX_TPLS`` # arguments/lists.) @@ -155,8 +155,8 @@ include(TribitsGeneralMacros) # ``TEST_OPTIONAL_PACKAGES``. # # The upstream dependencies within a single list do not need to be listed in -# any order. For example if ``PKG2`` depends on ``PKG1``, and this given -# package depends on both, then one can list:: +# any particular order. For example, if ``PKG2`` depends on ``PKG1``, and +# this given package depends on both, then one can list the dependencies as:: # # LIB_REQUIRED_PACKAGES PKG2 PKG1 # diff --git a/cmake/tribits/doc/guides/TribitsGuidesBody.rst b/cmake/tribits/doc/guides/TribitsGuidesBody.rst index 210e4cc80583..c978a2d103ce 100644 --- a/cmake/tribits/doc/guides/TribitsGuidesBody.rst +++ b/cmake/tribits/doc/guides/TribitsGuidesBody.rst @@ -1589,13 +1589,13 @@ are defined before a Package's ``CMakeLists.txt`` file is processed: imply that all of the required subpackages will be enabled, only that the parent package will be processed). - .. _${PACKAGE_NAME}_ENABLE_${OPTIONAL_DEP_PACKAGE_NAME}: + .. _${PACKAGE_NAME}_ENABLE_${UPSTREAM_PACKAGE_NAME}: - ``${PACKAGE_NAME}_ENABLE_${OPTIONAL_DEP_PACKAGE_NAME}`` + ``${PACKAGE_NAME}_ENABLE_${UPSTREAM_PACKAGE_NAME}`` Set to ``ON`` if support for the optional `upstream`_ dependent package - ``${OPTIONAL_DEP_PACKAGE_NAME}`` is enabled in package - ``${PACKAGE_NAME}``. Here ``${OPTIONAL_DEP_PACKAGE_NAME}`` corresponds to + ``${UPSTREAM_PACKAGE_NAME}`` is enabled in package + ``${PACKAGE_NAME}``. Here ``${UPSTREAM_PACKAGE_NAME}`` corresponds to each optional upstream package listed in the ``LIB_OPTIONAL_PACKAGES`` and ``TEST_OPTIONAL_PACKAGES`` arguments to the `tribits_package_define_dependencies()`_ macro. @@ -1603,11 +1603,11 @@ are defined before a Package's ``CMakeLists.txt`` file is processed: **NOTE:** It is important that the CMake code in the package's ``CMakeLists.txt`` files key off of this variable and **not** the project-level variable - ``${PROJECT_NAME}_ENABLE_${OPTIONAL_DEP_PACKAGE_NAME}`` because the + ``${PROJECT_NAME}_ENABLE_${UPSTREAM_PACKAGE_NAME}`` because the package-level variable - ``${PACKAGE_NAME}_ENABLE_${OPTIONAL_DEP_PACKAGE_NAME}`` can be explicitly + ``${PACKAGE_NAME}_ENABLE_${UPSTREAM_PACKAGE_NAME}`` can be explicitly turned off by the user even through the packages ``${PACKAGE_NAME}`` and - ``${OPTIONAL_DEP_PACKAGE_NAME}`` are both enabled at the project level! + ``${UPSTREAM_PACKAGE_NAME}`` are both enabled at the project level! See `Support for optional package can be explicitly disabled`_. **NOTE:** This variable will also be set for required dependencies as well @@ -1648,12 +1648,12 @@ are defined in the top-level project scope before a Package's ``HAVE__`` Set to ``ON`` if support for optional upstream package - ``${OPTIONAL_DEP_PACKAGE}`` is enabled in downstream package + ``${UPSTREAM_PACKAGE_NAME`` is enabled in downstream package ``${PACKAGE_NAME}`` - (i.e. `${PACKAGE_NAME}_ENABLE_${OPTIONAL_DEP_PACKAGE_NAME}`_ = ``ON``) and - is set to ``FALSE`` otherwise. Here, ```` and + (i.e. `${PACKAGE_NAME}_ENABLE_${UPSTREAM_PACKAGE_NAME}`_ = ``ON``) and is + set to ``FALSE`` otherwise. Here, ```` and ```` are the upper-case names for the packages - ``${PACKAGE_NAME}`` and ``${OPTIONAL_DEP_PACKAGE_NAME}``, respectively. + ``${PACKAGE_NAME}`` and ``${UPSTREAM_PACKAGE_NAME}``, respectively. For example, if optional support for upstream package ``Triutils`` is enabled in downstream package ``EpetraExt`` in `ReducedMockTrilinos`_, then ``EpetraExt_ENABLE_TriUtils=ON`` and ``HAVE_EPETRAEXT_TRIUTILS=ON``. @@ -1668,7 +1668,7 @@ are defined in the top-level project scope before a Package's #cmakedefine HAVE_EPETRAEXT_TRIUTILS NOTE: TriBITS automatically sets this variable depending on the value of - `${PACKAGE_NAME}_ENABLE_${OPTIONAL_DEP_PACKAGE_NAME}`_ during the step + `${PACKAGE_NAME}_ENABLE_${UPSTREAM_PACKAGE_NAME}`_ during the step "Adjust package and TPLs enables and disables" in `Full Processing of TriBITS Project Files`_. And tweaking this variable after that must be done carefully as described in `How to tweak downstream TriBITS "ENABLE"