From fa761dcc334d8f59af8a69556fd0909be39a7c02 Mon Sep 17 00:00:00 2001 From: Anderson Chauphan Date: Tue, 30 Jan 2024 16:16:59 -0700 Subject: [PATCH] Renamed variables and cache files (#600) Renamed variables and cache files to be shorter in cases where the expected source or direction is related to CDash or CDash tests. --- ...onPassingTests_2018-10-28_2018-10-28.json} | 0 ...onPassingTests_2018-10-28_2018-10-28.json} | 0 ...onPassingTests_2018-10-28_2018-10-28.json} | 0 ...ze_and_report_random_failures_UnitTests.py | 6 +- ...dash_analyze_and_report_random_failures.py | 68 +++++++++++-------- 5 files changed, 42 insertions(+), 32 deletions(-) rename test/ci_support/cdash_analyze_and_report_random_failures/not_rand_3pass_2fail/test_queries_cache/{fullCDashNonPassingTests_2018-10-28_2018-10-28.json => initialCDashNonPassingTests_2018-10-28_2018-10-28.json} (100%) rename test/ci_support/cdash_analyze_and_report_random_failures/rand_1pass_1fail/test_queries_cache/{fullCDashNonPassingTests_2018-10-28_2018-10-28.json => initialCDashNonPassingTests_2018-10-28_2018-10-28.json} (100%) rename test/ci_support/cdash_analyze_and_report_random_failures/random/test_queries_cache/{fullCDashNonPassingTests_2018-10-28_2018-10-28.json => initialCDashNonPassingTests_2018-10-28_2018-10-28.json} (100%) diff --git a/test/ci_support/cdash_analyze_and_report_random_failures/not_rand_3pass_2fail/test_queries_cache/fullCDashNonPassingTests_2018-10-28_2018-10-28.json b/test/ci_support/cdash_analyze_and_report_random_failures/not_rand_3pass_2fail/test_queries_cache/initialCDashNonPassingTests_2018-10-28_2018-10-28.json similarity index 100% rename from test/ci_support/cdash_analyze_and_report_random_failures/not_rand_3pass_2fail/test_queries_cache/fullCDashNonPassingTests_2018-10-28_2018-10-28.json rename to test/ci_support/cdash_analyze_and_report_random_failures/not_rand_3pass_2fail/test_queries_cache/initialCDashNonPassingTests_2018-10-28_2018-10-28.json diff --git a/test/ci_support/cdash_analyze_and_report_random_failures/rand_1pass_1fail/test_queries_cache/fullCDashNonPassingTests_2018-10-28_2018-10-28.json b/test/ci_support/cdash_analyze_and_report_random_failures/rand_1pass_1fail/test_queries_cache/initialCDashNonPassingTests_2018-10-28_2018-10-28.json similarity index 100% rename from test/ci_support/cdash_analyze_and_report_random_failures/rand_1pass_1fail/test_queries_cache/fullCDashNonPassingTests_2018-10-28_2018-10-28.json rename to test/ci_support/cdash_analyze_and_report_random_failures/rand_1pass_1fail/test_queries_cache/initialCDashNonPassingTests_2018-10-28_2018-10-28.json diff --git a/test/ci_support/cdash_analyze_and_report_random_failures/random/test_queries_cache/fullCDashNonPassingTests_2018-10-28_2018-10-28.json b/test/ci_support/cdash_analyze_and_report_random_failures/random/test_queries_cache/initialCDashNonPassingTests_2018-10-28_2018-10-28.json similarity index 100% rename from test/ci_support/cdash_analyze_and_report_random_failures/random/test_queries_cache/fullCDashNonPassingTests_2018-10-28_2018-10-28.json rename to test/ci_support/cdash_analyze_and_report_random_failures/random/test_queries_cache/initialCDashNonPassingTests_2018-10-28_2018-10-28.json diff --git a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py index 057128218..ec59cdbd1 100644 --- a/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py +++ b/test/ci_support/cdash_analyze_and_report_random_failures_UnitTests.py @@ -198,7 +198,7 @@ def test_base(self): expectedRtnCode=0, stdoutRegexList=[ "[*][*][*] CDash random failure analysis for ProjectName from 2018-10-28 to 2018-10-28", - "Total number of failing tests: 1" + "Total number of initial failing tests: 1" ], ) @@ -212,7 +212,7 @@ def test_rand_1pass_1fail(self): stdoutRegexList= [ "[*][*][*] CDash random failure analysis for ProjectName from 2018-10-28 to 2018-10-28", - "Total number of failing tests: 1", + "Total number of initial failing tests: 1", "Found randomly failing tests: 1", "Test name: testname1", @@ -237,7 +237,7 @@ def test_not_rand_3pass_2fail(self): "\s+Size of test history: 5", "[*][*][*] CDash random failure analysis for ProjectName from 2018-10-28 to 2018-10-28", - "Total number of failing tests: 1", + "Total number of initial failing tests: 1", "Found randomly failing tests: 0", ], diff --git a/tribits/ci_support/cdash_analyze_and_report_random_failures.py b/tribits/ci_support/cdash_analyze_and_report_random_failures.py index 9cc4c005f..229070fb1 100755 --- a/tribits/ci_support/cdash_analyze_and_report_random_failures.py +++ b/tribits/ci_support/cdash_analyze_and_report_random_failures.py @@ -43,28 +43,30 @@ def main(): args = getCmndLineArgs() cdashProjectTestingDayStartTime = "00:00" + # TODO: This should be moved outside in a project specific + # driver script or command line input cdashSiteUrl = args.cdash_site_url cdashProjectName = args.cdash_project_name date = args.reference_date groupName = args.group_name - # cdashNonpassedTestsFilters = args.cdash_nonpassed_tests_filters daysOfHistory = args.days_of_history printUrlMode = args.print_url_mode randomFailureSummaries = [] - cdashNonpassedTestsFilters = \ + cdashInitialNonpassedTestsFilters = \ "filtercount=2&showfilters=1&filtercombine=and"+\ "&field1=status&compare1=63&value1=Failed"+\ "&field2=groupname&compare2=63&value2=Pull%20Request" + # TODO: This cdashNonpassedTestsFilters should be moved outside + # and into a project-specific driver script or taken as a + # command line input. - # A) Setup directories and + # A) Set up date range and directories # Construct date range for queryTests filter string referenceDateDT = CDQAR.convertInputDateArgToYYYYMMDD( - cdashProjectTestingDayStartTime, date - ) - + cdashProjectTestingDayStartTime, date) dateRangeStart, dateRangeEnd = getDateRangeTuple(referenceDateDT, daysOfHistory) dateUrlField = "begin="+dateRangeStart+"&end="+dateRangeEnd @@ -76,36 +78,41 @@ def main(): createDirsFromPath(testHistoryCacheDir) # Construct queryTest.php filter for a date range - failingTestQueryFilters = \ - dateUrlField+"&"+cdashNonpassedTestsFilters + initialNonpassingTestQueryFilters = \ + dateUrlField+"&"+cdashInitialNonpassedTestsFilters # B.1) Get all failing test result for past daysOfHistory print("\nGetting list of nonpassing tests from CDash ...") - cdashNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( - cdashSiteUrl, cdashProjectName, None, failingTestQueryFilters) + initialNonpassingTestsQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( + cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) if printUrlMode == 'initial' or printUrlMode == 'all': - cdashNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( - cdashSiteUrl, cdashProjectName, None, failingTestQueryFilters) + initialNonpassingTestBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( + cdashSiteUrl, cdashProjectName, None, initialNonpassingTestQueryFilters) print("\nCDash nonpassing tests browser URL:\n\n"+\ - " "+cdashNonpassingTestBrowserUrl+"\n") + " "+initialNonpassingTestBrowserUrl+"\n") print("\nCDash nonpassing tests query URL:\n\n"+\ - " "+cdashNonpassingTestsQueryUrl+"\n") + " "+initialNonpassingTestsQueryUrl+"\n") - cdashNonpassingTestsQueryJsonCacheFile = \ - cdashQueriesCacheDir+"/fullCDashNonPassingTests_"+dateRangeStart+"_"+dateRangeEnd+".json" + initialNonpassingTestsQueryCacheFile = \ + cdashQueriesCacheDir+"/initialCDashNonPassingTests_"+dateRangeStart+"_"+dateRangeEnd+".json" # List of dictionaries containing the cdash results in rows - nonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( - cdashNonpassingTestsQueryUrl, cdashNonpassingTestsQueryJsonCacheFile,\ + initialNonpassingTestsLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( + initialNonpassingTestsQueryUrl, initialNonpassingTestsQueryCacheFile,\ alwaysUseCacheFileIfExists=True) # B.2) Get each nonpassing test's testing history - for nonpassingTest in nonpassingTestsLOD: + for nonpassingTest in initialNonpassingTestsLOD: + # Remove unique jenkins run ID from build name correctedBuildName = nonpassingTest['buildName'].rsplit('-', 1)[0] + # NOTE: This is project specific code. As Ross pointed out, buildName + # contains a lot of Trilinos specific prefix and suffix that should + # be processed by a Strategy class to be more project agnostic. + buildNameMax = 80 shortenedBuildName = correctedBuildName[:buildNameMax] @@ -118,40 +125,44 @@ def main(): "&field1=testname&compare1=63&value1="+nonpassingTest['testname']+\ "&field2=groupname&compare2=63&value2="+groupName+\ "&field3=buildname&compare3=63&value3="+correctedBuildName + testHistoryQueryFilters = dateUrlField+"&"+cdashTestHistoryFilters cdashTestHistoryCacheFile = testHistoryCacheDir+"/"+nonpassingTest['testname']+"_"+shortenedBuildName+".json" print("\n Creating file to write test history:\n "+cdashTestHistoryCacheFile) - cdashTestHistoryQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( + testHistoryQueryUrl = CDQAR.getCDashQueryTestsQueryUrl( cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters) - cdashTestHistoryBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( + testHistoryBrowserUrl = CDQAR.getCDashQueryTestsBrowserUrl( cdashSiteUrl, cdashProjectName, None, testHistoryQueryFilters) testHistoryLOD = CDQAR.downloadTestsOffCDashQueryTestsAndFlatten( - cdashTestHistoryQueryUrl, cdashTestHistoryCacheFile, alwaysUseCacheFileIfExists=True) + testHistoryQueryUrl, cdashTestHistoryCacheFile, alwaysUseCacheFileIfExists=True) print("\n Size of test history: "+str(len(testHistoryLOD))) if printUrlMode == 'all': print("\n CDash test history browser URL for "+nonpassingTest['testname']+" "+\ - correctedBuildName+":\n\n"+" "+cdashTestHistoryBrowserUrl) + correctedBuildName+":\n\n"+" "+testHistoryBrowserUrl) print("\n CDash test history query URL for "+nonpassingTest['testname']+" "+\ - correctedBuildName+":\n\n"+" "+cdashTestHistoryQueryUrl) + correctedBuildName+":\n\n"+" "+testHistoryQueryUrl) if len(testHistoryLOD) < 2: + print("\n Size of test history too small for any comparisons, skipping ...\n") continue - # B.2a) Split full testing history to passed and nonpassed lists of dicts + # B.3) Split full testing history to passed and nonpassed lists of dicts passingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Passed"] nonpassingTestHistoryLOD = [test for test in testHistoryLOD if test.get("status") == "Failed"] nonpassingSha1Pairs = set() - print("\n Failed tests: "+str(len(nonpassingTestHistoryLOD))) - print("\n Passing tests: "+str(len(passingTestHistoryLOD))) + print("\n Num of passing tests in test history: "+str(len(passingTestHistoryLOD))) + print("\n Num of nonpassing tests in test history: "+str(len(nonpassingTestHistoryLOD))) buildSummaryCacheDir = cdashQueriesCacheDir+"/build_summary_cache/"+nonpassingTest['testname']+"_"+shortenedBuildName createDirsFromPath(buildSummaryCacheDir) + # NOTE: There is an argumåent to be made that test histories should get their own directory + # instead of build summaries and that build summaries should live inside of there # C.1) Get all nonpassing tests' sha1s into a set for test in nonpassingTestHistoryLOD: @@ -192,7 +203,7 @@ def main(): print("\n*** CDash random failure analysis for "+cdashProjectName+" from " +dateRangeStart+" to "+dateRangeEnd) - print("Total number of initial failing tests: "+str(len(nonpassingTestsLOD))+"\n") + print("Total number of initial failing tests: "+str(len(initialNonpassingTestsLOD))+"\n") print("Found randomly failing tests: "+str(len(randomFailureSummaries))) for summary in randomFailureSummaries: @@ -208,7 +219,6 @@ def getCmndLineArgs(): parser.add_argument("--cdash-project-name", default="", required=True) parser.add_argument("--reference-date", default="yesterday") parser.add_argument("--group-name", default="Pull%20Request") - # parser.add_argument("--cdash-nonpassed-tests-filters", default="") parser.add_argument("--days-of-history", default=1, type=int) parser.add_argument("--print-url-mode", choices=['none','initial','all'], default='none')