Enable filtering of flow jobs/runs, feature enhancement and bug fixes. (

#10)

* Elapsed Time Support.

* Elapsed Support

* Make changes to match Jared and Daniel's

* Stable Elapsed Time View.

* Add the view to show the sum of elapsed time of all jobs, for each repo.

* Enabled the "Job Elapsed Time List" page for each repo

* Make JobET view full function.

* Job ET per build view is complete

* Use public URL instead of debugging one, so it'll be accessible by others.

* Use @Url.Action instead of a hardcoded URI (#3)

Url.Action(...)  is a built-in MVC helper function that can build up a URI regardless of where the website is running. This allows the code to work regardless of where the webpage is deployed ( localhost ,  jdash-et ,  jdash , etc).

* 1. Add support to make bootstrap table columns sortable.
2. Make "Job ET per build" table sortable, so managers can choose to sort by either "Build Number", and "Elapsed Time".

* Add "ElapsedTime" in the main layout as the entrance of Elapsed Time Report.

* Add report on average ET per job and average ET per build (of a job).

* Changes to address Jared's comments.

* Make table on "Job Elapsed Time List" view sortable.

* Disable dropdown list from "Job Elapsed Time per build“ view.

* Change to show elapsed time in minutes instead of seconds to keep consistency.

* Fixed table sorting issue on "Job Elapsed Time List" view, caused ty inappropriate data type.

* UI Work to add a check box on views below to allow users to include/exclude elapsed time results from flow jobs/runs:
• “Total Elapsed Time List of all Projects” view
• “Job Elapsed Time List” view

* Fixed the issue below:
When too many project names are loaded in "Total Elapsed Time List of all Projects" view, some of them disappear.

* Make table on "Job Elapsed Time List" view sortable.

* Disable dropdown list from "Job Elapsed Time per build“ view.

* Change to show elapsed time in minutes instead of seconds to keep consistency.

* Fixed table sorting issue on "Job Elapsed Time List" view, caused ty inappropriate data type.

* UI Work to add a check box on views below to allow users to include/exclude elapsed time results from flow jobs/runs:
• “Total Elapsed Time List of all Projects” view
• “Job Elapsed Time List” view

* Fixed the issue below:
When too many project names are loaded in "Total Elapsed Time List of all Projects" view, some of them disappear.

* Not showing flow job/runs as default.

* Older runs (before the DB column was added) don't have the field of "JobKind" yet, so checking its existence first to avoid exception.

* Google chart has bugs when dealing checked checkbox as default. As a result, I have to make the check box starting from an "unchecked" state.

* Address all review comments from Jared.

ApiFun/Program.cs

@@ -95,7 +95,7 @@ private static async Task FillData()
 
         private static async Task TestJob()
         {
-            var jobUrlStr = "http://dotnet-ci.cloudapp.net/job/Private/job/dotnet_roslyn-internal/job/microupdate/job/windows_vsi_p2/2/";
+            var jobUrlStr = "http://dotnet-ci.cloudapp.net/job/Private/job/dotnet_roslyn-internal/job/microupdate/job/windows_vsi_p2/8/";
             var uri = new Uri(jobUrlStr);
             var parts = uri.PathAndQuery.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
             var jobPath = string.Join("/", parts.Take(parts.Length - 1));
@@ -354,7 +354,7 @@ private static async Task TestPopulator()
             var client = CreateClient(auth: false);
             var populator = new BuildTablePopulator(account.CreateCloudTableClient(), client, Console.Out);
 
-            var boundBuildId = BoundBuildId.Parse("https://dotnet-ci.cloudapp.net/job/dotnet_corefx/job/master/job/fedora23_debug_tst/134/");
+            var boundBuildId = BoundBuildId.Parse("https://dotnet-ci.cloudapp.net/job/dotnet_coreclr/job/master/job/jitstress/job/x64_checked_osx_jitstress1_flow/7/");
             try
             {
                 await populator.PopulateBuild(boundBuildId.BuildId);

Dashboard/Controllers/BuildsController.cs

@@ -13,6 +13,7 @@
 using System.Web.Mvc;
 using Dashboard.Helpers;
 using System.Text;
+using System.Xml;
 
 namespace Dashboard.Controllers
 {
@@ -149,14 +150,15 @@ public ActionResult ElapsedTime(bool pr = false, DateTimeOffset? startDate = nul
         /// A view of the total elapsed time per team/project repo, ranked from most elapsed time to least.
         /// </summary>
         /// <returns></returns>
-        public ActionResult ProjectElapsedTime(bool pr = false, DateTimeOffset? startDate = null)
+        public ActionResult ProjectElapsedTime(bool pr = false, bool fr = false, DateTimeOffset? startDate = null)
         {
-            var filter = CreateBuildFilter(actionName: nameof(ProjectElapsedTime), startDate: startDate, pr: pr);
+            var filter = CreateBuildFilter(actionName: nameof(ProjectElapsedTime), startDate: startDate, pr: pr, fr: fr, displayFRCheckBox: true);
 
             List<string> repoNameList = _buildUtil.GetViewNames(filter.StartDate);
             List<ProjectElapsedTimeModel> ETListOfProjects = new List<ProjectElapsedTimeModel>();
             var totalCount = 0;
             var totalSucceeded = 0;
+            var totalFlowJobCount = 0;
 
             foreach (var repoName in repoNameList)
             {
@@ -169,19 +171,35 @@ public ActionResult ProjectElapsedTime(bool pr = false, DateTimeOffset? startDat
                     .Where(x => pr || !JobUtil.IsPullRequestJobName(x.JobId))
                     .ToList();
 
-                if (repoName == AzureUtil.ViewNameAll)
-                {
-                    totalCount = results.Count;
-                    totalSucceeded = results.Count(x => x.ClassificationKind == ClassificationKind.Succeeded);
-                }
-
                 var runCounts = results
-                    .Select(x => new ElapsedTimeModel() { JobId = x.JobId, JobName = x.JobName, ElapsedTime = x.DurationSeconds })
+                    .Select(x => new ElapsedTimeModel() { JobId = x.JobId, JobName = x.JobName, ElapsedTime = x.DurationSeconds, ClassificationKind = x.ClassificationKind, JobKind = x.JobKind })
                     .ToList();
 
                 foreach (var runElapsedTime in runCounts)
                 {
+                    //If users choose to exclude flow run results
+                    if (!filter.IncludeFlowRunResults)
+                    {
+                        if (runElapsedTime.JobKind != null && runElapsedTime.JobKind.Equals(JobKind.Flow))
+                        {
+                            //To avoid double count on total # of flow jobs.
+                            if (repoName == AzureUtil.ViewNameAll)
+                            {
+                                totalFlowJobCount++;
+                            }
+                            continue;
+                        }
+                    }
+
                     currRepo.ETSum = currRepo.ETSum + runElapsedTime.ElapsedTime;
+
+                    if (repoName == AzureUtil.ViewNameAll)
+                    {
+                        totalCount++;
+
+                        if (runElapsedTime.ClassificationKind == ClassificationKind.Succeeded)
+                            totalSucceeded++;
+                    }
                 }
 
                 //Store total elapsed time in minutes.
@@ -194,6 +212,7 @@ public ActionResult ProjectElapsedTime(bool pr = false, DateTimeOffset? startDat
             {
                 Filter = filter,
                 TotalBuildCount = totalCount,
+                FlowJobCount = totalFlowJobCount,
                 TotalSucceededCount = totalSucceeded,
                 ProjectElapsedTimeList = ETListOfProjects
             };
@@ -240,27 +259,29 @@ public ActionResult KindByViewName(string name = null, bool pr = false, DateTime
             return View(viewName: "KindByViewName", model: model);
         }
 
-        public ActionResult JobListByRepoName(string name = null, bool pr = false, DateTime? startDate = null, string viewName = AzureUtil.ViewNameAll)
+        public ActionResult JobListByRepoName(string name = null, bool pr = false, bool fr = false, DateTime? startDate = null, string viewName = AzureUtil.ViewNameAll)
         {
             var startDateValue = startDate ?? DateTimeOffset.UtcNow - TimeSpan.FromDays(1);
             BuildFilterModel filter;
             List<BuildResultEntity> results;
             var totalJobCount = 0;
+            var totalRunCount = 0;
             var totalETOfCurrRepo = 0;
+            var totalFlowRunCount = 0;
 
             //When navigating from "ProjectElapsedTime" view to "JobElapsedTime" view, var "name" is set to the repo name being selected.
             //When refreshing "JobElapsedTime" view via repo name dropdown list, var "viewName" is set to the repo name, var "name" == null
             if (name != null)
             {
-                filter = CreateBuildFilter(actionName: nameof(JobListByRepoName), viewName: name, startDate: startDate, pr: pr);
+                filter = CreateBuildFilter(actionName: nameof(JobListByRepoName), viewName: name, startDate: startDate, pr: pr, fr: fr, displayFRCheckBox: true);
                 results = _buildUtil
                     .GetBuildResults(startDateValue, name)
                     .Where(x => pr || !JobUtil.IsPullRequestJobName(x.JobId))
                     .ToList();
             }
             else
             {
-                filter = CreateBuildFilter(actionName: nameof(JobListByRepoName), viewName: viewName, startDate: startDate, pr: pr);
+                filter = CreateBuildFilter(actionName: nameof(JobListByRepoName), viewName: viewName, startDate: startDate, pr: pr, fr: fr, displayFRCheckBox: true);
                 results = _buildUtil
                     .GetBuildResults(startDateValue, viewName)
                     .Where(x => pr || !JobUtil.IsPullRequestJobName(x.JobId))
@@ -270,6 +291,16 @@ public ActionResult JobListByRepoName(string name = null, bool pr = false, DateT
             SortedDictionary<string, AgJobElapsedTime> aggregatedJobElapsedTimeDic = new SortedDictionary<string, AgJobElapsedTime>();
             foreach (var entry in results)
             {
+                //If users choose to exclude flow run results
+                if (!filter.IncludeFlowRunResults)
+                {
+                    if (entry.JobKind != null && entry.JobKind.Equals(JobKind.Flow))
+                    {
+                        totalFlowRunCount++;
+                        continue;
+                    }
+                }
+
                 string currJobName = entry.BuildId.JobName;
                 totalETOfCurrRepo += entry.DurationSeconds;
 
@@ -287,19 +318,21 @@ public ActionResult JobListByRepoName(string name = null, bool pr = false, DateT
                 }
             }
 
-            totalJobCount = aggregatedJobElapsedTimeDic.Count;
+            totalRunCount = results.Count - totalFlowRunCount;
+            totalJobCount += aggregatedJobElapsedTimeDic.Count;
 
             var model = new JobElapsedTimeModel()
             {
                 Filter = filter,
                 TotalJobCount = totalJobCount,
+                TotalRunCount = totalRunCount,
+                FlowRunCount = totalFlowRunCount,
                 TotalETOfCurrRepo = totalETOfCurrRepo,
                 AgJobElapsedTimeDict = aggregatedJobElapsedTimeDic
             };
             return View(viewName: "JobElapsedTime", model: model);
         }
 
-
         public ActionResult JobElapsedTimePerBuild(bool pr = false, DateTime? startDate = null, string viewName = AzureUtil.ViewNameRoslyn, string jobName = "dotnet_coreclr/master/checked_windows_nt_bld")
         {
             var startDateValue = startDate ?? DateTimeOffset.UtcNow - TimeSpan.FromDays(1);
@@ -326,8 +359,7 @@ public ActionResult JobElapsedTimePerBuild(bool pr = false, DateTime? startDate
 
             return View(viewName: "JobElapsedTimePerBuild", model: model);
         }
-
-
+
         public string Csv(string viewName = AzureUtil.ViewNameRoslyn, bool pr = false, DateTime? startDate = null)
         {
             var filter = CreateBuildFilter(nameof(Csv), viewName: viewName, pr: pr, startDate: startDate);
@@ -460,13 +492,15 @@ private TestFailureModel GetTestFailureModel(BuildFilterModel filter)
             return model;
         }
 
-        private BuildFilterModel CreateBuildFilter(string actionName, string name = null, string viewName = null, bool pr = false, DateTimeOffset? startDate = null, int? limit = null)
+        private BuildFilterModel CreateBuildFilter(string actionName, string name = null, string viewName = null, bool pr = false, DateTimeOffset? startDate = null, int? limit = null, bool fr = false, bool displayFRCheckBox = false)
         {
             return new BuildFilterModel()
             {
                 Name = name,
                 ViewName = viewName,
                 IncludePullRequests = pr,
+                IncludeFlowRunResults = fr,
+                DisplayFlowRunCheckBox = displayFRCheckBox,
                 StartDate = startDate ?? DateTimeOffset.UtcNow - TimeSpan.FromDays(1),
                 Limit = limit,
                 ActionName = actionName,

Dashboard/Models/BuildsModel.cs

@@ -11,6 +11,13 @@ namespace Dashboard.Models
     public sealed class BuildFilterModel
     {
         public bool IncludePullRequests { get; set; }
+        /// <summary>
+        /// Whether to include results from flow jobs/runs
+        /// The elapsed time of a flow job/run is the sum of elapsed time of all its sub jobs/runs.
+        /// They should be excluded from elapsed time calcuation, as they do NOT consume additional machine resources.
+        /// </summary>
+        public bool IncludeFlowRunResults { get; set; }
+        public bool DisplayFlowRunCheckBox { get; set; }
         public DateTimeOffset StartDate { get; set; }
         public int? Limit { get; set; }
         public string Name { get; set; }
@@ -159,7 +166,9 @@ public class ElapsedTimeModel
     {
         public JobId JobId { get; set; }
         public string JobName { get; set; }
+        public string JobKind { get; set; }
         public int ElapsedTime { get; set; }
+        public Dashboard.Azure.ClassificationKind ClassificationKind { get; set; }
     }
 
     /// <summary>
@@ -212,6 +221,11 @@ public class ProjectElapsedTimeSummaryModel
         /// </summary>
         public int TotalBuildCount { get; set; }
 
+        /// <summary>
+        /// Total number of flow jobs.
+        /// </summary>
+        public int FlowJobCount { get; set; }
+
         /// <summary>
         /// Total number of builds that succeeded.
         /// </summary>
@@ -248,6 +262,18 @@ public class JobElapsedTimeModel
         /// </summary>
         public int TotalJobCount { get; set; }
 
+        /// <summary>
+        /// Total number of runs
+        /// Note even though each job can have multiple runs/builds
+        /// If "Include Flow Run/Job Results" are not checked, the # of flow runs will NOT be counted.
+        /// </summary>
+        public int TotalRunCount { get; set; }
+
+        /// <summary>
+        /// Total number of flow runs.
+        /// </summary>
+        public int FlowRunCount { get; set; }
+
         /// <summary>
         /// Total elapsed time of current repo
         /// </summary>

Dashboard/Views/Builds/BuildFilter.cshtml

@@ -2,6 +2,8 @@
 
 @{
     var prValue = Model.IncludePullRequests ? @"checked=""checked""" : "";
+    var frValue = Model.IncludeFlowRunResults ? @"checked=""checked""" : "";
+    var showFRBox = Model.DisplayFlowRunCheckBox;
     var startDateValue = Model.StartDate.ToString("yyyy-MM-dd");
 }
 <h2>Filter Results</h2>
@@ -20,6 +22,10 @@
 
     <div>
         <div>Include Pull Requests <input name="pr" type="checkbox" @prValue value="true" /></div>
+        @if (showFRBox)
+        {
+            <div>Include Flow Run/Job Results <input name="fr" type="checkbox" @frValue value="true" /></div>
+        }
         <div>Start Date <input name="startDate" type="date" value="@startDateValue"/></div>
         @if (Model.ViewName != null)
         {

Dashboard/Views/Builds/JobElapsedTime.cshtml

@@ -5,23 +5,30 @@
     ViewBag.Title = "Job Elapsed Time List";
     var showForPrStyle = Model.Filter.IncludePullRequests ? "" : "display: none";
     var prValue = Model.Filter.IncludePullRequests ? @"checked=""checked""" : "";
+    var frValue = Model.Filter.IncludeFlowRunResults ? @"checked=""checked""" : "";
     var startDateValue = Model.Filter.StartDate.ToString("yyyy-MM-dd");
-    var aveET = Model.TotalETOfCurrRepo / (double)Model.TotalJobCount;
+    var totalETInMin = (double)Model.TotalETOfCurrRepo / 60;
+    var aveET = (double)totalETInMin / Model.TotalJobCount;
 }
 
 <div>
     <h2>Job Elapsed Time List</h2>
     <div>Jenkins Jobs (@Model.Filter.ViewName)</div>
     <div>Job count: @Model.TotalJobCount.ToString("n0")</div>
-    <div>Total elapsed time (in secs) @Model.TotalETOfCurrRepo.ToString("n0")</div>
-    <div>Average elapsed time per job (in secs) @aveET.ToString("n2")</div>
+    <div>Run count: @Model.TotalRunCount.ToString("n0")</div>
+    @if (!@Model.Filter.IncludeFlowRunResults)
+    {
+        <div>Flow run count: @Model.FlowRunCount.ToString("n0")</div>
+    }
+    <div>Total elapsed time (in minutes) @totalETInMin.ToString("n2")</div>
+    <div>Average elapsed time per job (in minutes) @aveET.ToString("n2")</div>
 </div>
 
-<table class="table">
+<table class="table sortable">
     <thead>
         <tr>
             <th>Job Name</th>
-            <th>Aggregated Elapsed Time (in secs)</th>
+            <th>Aggregated Elapsed Time (in minutes)</th>
             <th>Number of runs of current job</th>
         </tr>
     </thead>
@@ -34,9 +41,10 @@
                 startDate = startDateValue,
                 jobName = currJobElapsedTime.Key
             });
+            var ETSumInMin = Math.Round((double)currJobElapsedTime.Value.ETSum / 60, 2);
             <tr>
                 <td>@currJobElapsedTime.Key</td>
-                <td>@currJobElapsedTime.Value.ETSum</td>
+                <td>@ETSumInMin</td>
                 <td><a href="@uri">@currJobElapsedTime.Value.NumOfBuilds</a></td>
             </tr>
         }
@@ -54,6 +62,7 @@
 {
     <input id="category_form_kind" name="name" value="" hidden="hidden" />
     <input name="pr" type="checkbox" @prValue value="true" hidden="hidden" />
+    <input name="fr" type="checkbox" @frValue value="true" hidden="hidden" />
     <input name="startDate" type="date" value="@startDateValue" hidden="hidden" />
     <input name="viewName" value="@Model.Filter.ViewName" hidden="hidden" />
 }

Dashboard/Views/Builds/JobElapsedTimePerBuild.cshtml

@@ -4,15 +4,18 @@
 @{
     ViewBag.Title = "Job Elapsed Time per build";
     var showForPrStyle = Model.Filter.IncludePullRequests ? "" : "display: none";
-    var aveET = Model.TotalETOfCurrJob / (double)Model.TotalBuildCount;
+    var totalETInMins = (double)Model.TotalETOfCurrJob / 60;
+    var aveET = (double)totalETInMins / Model.TotalBuildCount;
+    var currView = Model.Filter.ViewName;
+    Model.Filter.ViewName = null;
 }
 
 <div>
     <h2>Job Elapsed Time per build</h2>
-    <div>Job builds of repo (@Model.Filter.ViewName)</div>
+    <div>Job builds of repo (@currView)</div>
     <div>Build/run count of current job: @Model.TotalBuildCount.ToString("n0")</div>
-    <div>Total elapsed time (in secs) @Model.TotalETOfCurrJob.ToString("n0")</div>
-    <div>Average elapsed time per build/run (in secs) @aveET.ToString("n2")</div>
+    <div>Total elapsed time (in minutes) @totalETInMins.ToString("n2")</div>
+    <div>Average elapsed time per build/run (in minutes) @aveET.ToString("n2")</div>
 </div>
 
 <table class="table sortable">
@@ -33,10 +36,11 @@
             var id = entity.BuildId;
             var uri = JenkinsUtil.GetUri(SharedConstants.DotnetJenkinsUri, JenkinsUtil.GetBuildPath(id));
             var prUri = entity.PullRequestUrl ?? "";
+            var ETInMins = (double)entity.DurationSeconds / 60;
             <tr>
                 <td>@id.JobName</td>
                 <td><a href="@uri">@id.Number</a></td>
-                <td>@entity.DurationSeconds</td>
+                <td>@ETInMins.ToString("n2")</td>
                 <td>@entity.MachineName</td>
                 <td>@entity.BuildDateTime.ToLocalTime().ToString("MM/dd hh:mm tt")</td>
                 <td style="@showForPrStyle">@entity.PullRequestAuthor</td>

Dashboard/Views/Builds/ProjectElapsedTime.cshtml

@@ -6,6 +6,7 @@
     var ETList = Model.ProjectElapsedTimeList.OrderByDescending(x => x.ETSum);
     var rankedETList = string.Join(";", ETList.Select(x => $"{x.RepoName},{x.ETSum}"));
     var prValue = Model.Filter.IncludePullRequests ? @"checked=""checked""" : "";
+    var frValue = Model.Filter.IncludeFlowRunResults ? @"checked=""checked""" : "";
     var startDateValue = Model.Filter.StartDate.ToString("yyyy-MM-dd");
     var successRate = Model.TotalSucceededCount / (double)Model.TotalBuildCount;
     var isSingleViewName = false;
@@ -15,11 +16,15 @@
     <h2>Total Elapsed Time List of all Projects</h2>
     <div>@Model.Filter.ViewName</div>
     <div>Ran @Model.TotalBuildCount.ToString("n0")</div>
+    @if (!@Model.Filter.IncludeFlowRunResults)
+    {
+        <div>Flow job count: @Model.FlowJobCount.ToString("n0")</div>
+    }
     <div>Succeded @Model.TotalSucceededCount.ToString("n0")</div>
     <div>Success rate @successRate.ToString("n2")</div>
 </div>
 
-<div id="repo_ET_chart" style="width: 900px; height:700px" data-values="@rankedETList"></div>
+<div id="repo_ET_chart" style="width: 900px; height:900px" data-values="@rankedETList"></div>
 
 <!-- Filter the action results -->
 @Html.Partial("BuildFilter", Model.Filter)
@@ -33,6 +38,7 @@
 {
     <input id="category_form_kind" name="name" value="" hidden="hidden" />
     <input name="pr" type="checkbox" @prValue value="true" hidden="hidden" />
+    <input name="fr" type="checkbox" @frValue value="true" hidden="hidden" />
     <input name="startDate" type="date" value="@startDateValue" hidden="hidden" />
     if (isSingleViewName)
     {