TT-1911 Fix 0% flakiness if ratio if only few tests were flaky (#1538)

tools/flakeguard/reports/data.go

@@ -2,7 +2,6 @@ package reports
 
 import (
 	"fmt"
-	"math"
 	"sort"
 	"strings"
 	"time"
@@ -121,6 +120,7 @@ type SplunkTestResultEvent struct {
 
 func GenerateSummaryData(tests []TestResult, maxPassRatio float64) SummaryData {
 	var runs, passes, fails, skips, panickedTests, racedTests, flakyTests, skippedTests int
+
 	for _, result := range tests {
 		runs += result.Runs
 		passes += result.Successes
@@ -143,30 +143,47 @@ func GenerateSummaryData(tests []TestResult, maxPassRatio float64) SummaryData {
 		}
 	}
 
+	// Calculate the raw pass ratio
 	passPercentage := 100.0
-	flakePercentage := 0.0
-
 	if runs > 0 {
-		passPercentage = math.Floor((float64(passes)/float64(runs)*100)*100) / 100 // Truncate to 2 decimal places
+		passPercentage = (float64(passes) / float64(runs)) * 100
 	}
 
+	// Calculate the raw flake ratio
 	totalTests := len(tests)
+	flakePercentage := 0.0
 	if totalTests > 0 {
-		flakePercentage = math.Floor((float64(flakyTests)/float64(totalTests)*100)*100) / 100 // Truncate to 2 decimal places
+		flakePercentage = (float64(flakyTests) / float64(totalTests)) * 100
 	}
 
+	// Helper function to convert a float ratio into a trimmed string
+	formatRatio := func(val float64) string {
+		// Format with 4 decimal places
+		s := fmt.Sprintf("%.4f", val)
+		// Trim trailing zeros
+		s = strings.TrimRight(s, "0")
+		// Trim trailing '.' if needed (in case we have an integer)
+		s = strings.TrimRight(s, ".")
+		return s + "%"
+	}
+
+	passRatioStr := formatRatio(passPercentage)
+	flakeTestRatioStr := formatRatio(flakePercentage)
+
 	return SummaryData{
 		TotalTests:     totalTests,
 		PanickedTests:  panickedTests,
 		RacedTests:     racedTests,
 		FlakyTests:     flakyTests,
-		FlakyTestRatio: fmt.Sprintf("%.2f%%", flakePercentage),
-		TotalRuns:      runs,
-		PassedRuns:     passes,
-		FailedRuns:     fails,
-		SkippedRuns:    skips,
-		PassRatio:      fmt.Sprintf("%.2f%%", passPercentage),
-		MaxPassRatio:   maxPassRatio,
+		FlakyTestRatio: flakeTestRatioStr,
+
+		TotalRuns:   runs,
+		PassedRuns:  passes,
+		FailedRuns:  fails,
+		SkippedRuns: skips,
+
+		PassRatio:    passRatioStr,
+		MaxPassRatio: maxPassRatio,
 	}
 }
 

tools/flakeguard/reports/data_test.go

@@ -28,12 +28,12 @@ func TestGenerateSummaryData(t *testing.T) {
 				PanickedTests:  0,
 				RacedTests:     0,
 				FlakyTests:     0,
-				FlakyTestRatio: "0.00%",
+				FlakyTestRatio: "0%", // no flaky tests
 				TotalRuns:      15,
 				PassedRuns:     15,
 				FailedRuns:     0,
 				SkippedRuns:    0,
-				PassRatio:      "100.00%",
+				PassRatio:      "100%",
 				MaxPassRatio:   1.0,
 			},
 		},
@@ -46,17 +46,19 @@ func TestGenerateSummaryData(t *testing.T) {
 			},
 			maxPassRatio: 0.9,
 			expected: SummaryData{
-				TotalTests:     3,
-				PanickedTests:  0,
-				RacedTests:     0,
-				FlakyTests:     2,
-				FlakyTestRatio: "66.66%",
+				TotalTests:    3,
+				PanickedTests: 0,
+				RacedTests:    0,
+				FlakyTests:    2,
+				// 2/3 => 66.666...%
+				FlakyTestRatio: "66.6667%",
 				TotalRuns:      19,
 				PassedRuns:     15,
-				FailedRuns:     4,
+				FailedRuns:     4, // total failures
 				SkippedRuns:    0,
-				PassRatio:      "78.94%",
-				MaxPassRatio:   0.9,
+				// 15/19 => ~78.947...
+				PassRatio:    "78.9474%",
+				MaxPassRatio: 0.9,
 			},
 		},
 		{
@@ -68,17 +70,19 @@ func TestGenerateSummaryData(t *testing.T) {
 			},
 			maxPassRatio: 1.0,
 			expected: SummaryData{
-				TotalTests:     3,
-				PanickedTests:  1,
-				RacedTests:     1,
-				FlakyTests:     2,
-				FlakyTestRatio: "66.66%",
+				TotalTests:    3,
+				PanickedTests: 1,
+				RacedTests:    1,
+				FlakyTests:    2,
+				// 2/3 => ~66.666...
+				FlakyTestRatio: "66.6667%",
 				TotalRuns:      18,
 				PassedRuns:     17,
 				FailedRuns:     1,
 				SkippedRuns:    0,
-				PassRatio:      "94.44%",
-				MaxPassRatio:   1.0,
+				// 17/18 => ~94.444...
+				PassRatio:    "94.4444%",
+				MaxPassRatio: 1.0,
 			},
 		},
 		{
@@ -90,13 +94,14 @@ func TestGenerateSummaryData(t *testing.T) {
 				PanickedTests:  0,
 				RacedTests:     0,
 				FlakyTests:     0,
-				FlakyTestRatio: "0.00%",
+				FlakyTestRatio: "0%", // no tests => 0%
 				TotalRuns:      0,
 				PassedRuns:     0,
 				FailedRuns:     0,
 				SkippedRuns:    0,
-				PassRatio:      "100.00%",
-				MaxPassRatio:   1.0,
+				// With zero runs, we default passRatio to "100%"
+				PassRatio:    "100%",
+				MaxPassRatio: 1.0,
 			},
 		},
 		{
@@ -110,13 +115,13 @@ func TestGenerateSummaryData(t *testing.T) {
 				TotalTests:     2,
 				PanickedTests:  0,
 				RacedTests:     0,
-				FlakyTests:     1,
-				FlakyTestRatio: "50.00%",
+				FlakyTests:     1,     // second test has ratio=0.7 => "flaky"
+				FlakyTestRatio: "50%", // 1 out of 2 => 50%
 				TotalRuns:      10,
 				PassedRuns:     7,
 				FailedRuns:     3,
-				SkippedRuns:    1,
-				PassRatio:      "70.00%",
+				SkippedRuns:    1, // from first test
+				PassRatio:      "70%",
 				MaxPassRatio:   0.8,
 			},
 		},
@@ -132,14 +137,53 @@ func TestGenerateSummaryData(t *testing.T) {
 				TotalTests:     3,
 				PanickedTests:  0,
 				RacedTests:     0,
+				FlakyTests:     1,          // last test has ratio=0.5 => "flaky"
+				FlakyTestRatio: "33.3333%", // 1 out of 3 => 33.333...
+				TotalRuns:      14,         // 10 + 4
+				PassedRuns:     11,         // 9 + 2
+				FailedRuns:     3,          // 1 + 2
+				SkippedRuns:    1,          // from first test
+				// 11/14 => 78.5714...
+				PassRatio:    "78.5714%",
+				MaxPassRatio: 0.85,
+			},
+		},
+		{
+			name: "Tiny flake ratio that is exactly 0.01%",
+			testResults: func() []TestResult {
+				// 9,999 total:
+				//  - 9,998 stable => pass=1.0
+				//  - 1 flaky => pass=0.5
+				const total = 9999
+				tests := make([]TestResult, total)
+				for i := 0; i < total-1; i++ {
+					tests[i] = TestResult{
+						PassRatio: 1.0,
+						Runs:      10,
+						Successes: 10,
+					}
+				}
+				tests[total-1] = TestResult{
+					PassRatio: 0.5, // 1 success, 1 fail
+					Runs:      2,
+					Successes: 1,
+					Failures:  1,
+				}
+				return tests
+			}(),
+			maxPassRatio: 1.0,
+			expected: SummaryData{
+				TotalTests:     9999,
+				PanickedTests:  0,
+				RacedTests:     0,
 				FlakyTests:     1,
-				FlakyTestRatio: "33.33%",
-				TotalRuns:      14,
-				PassedRuns:     11,
-				FailedRuns:     3,
-				SkippedRuns:    1,
-				PassRatio:      "78.57%",
-				MaxPassRatio:   0.85,
+				FlakyTestRatio: "0.01%",
+				TotalRuns:      (9998 * 10) + 2,
+				PassedRuns:     (9998 * 10) + 1,
+				FailedRuns:     1,
+				SkippedRuns:    0,
+				PassRatio:      "99.999%",
+				MaxPassRatio:   1.0,
 			},
 		},
 	}
@@ -148,7 +192,8 @@ func TestGenerateSummaryData(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			summary := GenerateSummaryData(tc.testResults, tc.maxPassRatio)
 			if !reflect.DeepEqual(summary, tc.expected) {
-				t.Errorf("Test %s failed. Expected %+v, got %+v", tc.name, tc.expected, summary)
+				t.Errorf("Test %q failed.\nExpected: %+v\nGot:      %+v",
+					tc.name, tc.expected, summary)
 			}
 		})
 	}

tools/flakeguard/reports/presentation_test.go

@@ -253,15 +253,15 @@ func TestRenderResults(t *testing.T) {
 				PanickedTests:  0,
 				RacedTests:     0,
 				FlakyTests:     1,
-				FlakyTestRatio: "100.00%",
+				FlakyTestRatio: "100%",
 				TotalRuns:      4,
 				PassedRuns:     3,
 				FailedRuns:     1,
 				SkippedRuns:    0,
-				PassRatio:      "75.00%",
+				PassRatio:      "75%",
 				MaxPassRatio:   0.9,
 			},
-			expectedStringsContain: []string{"Test1", "package1", "75.00%", "false", "1.05s", "4", "0"},
+			expectedStringsContain: []string{"Test1", "package1", "75%", "false", "1.05s", "4", "0"},
 		},
 		// Add more test cases as needed
 	}