Skip to content

Commit

Permalink
Keep test outputs for Flakeguard in separate fields (#1485)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lukaszcl
lukaszcl authored Dec 16, 2024
1 parent 1421519 commit ea4ffd8
Show file tree
Hide file tree
Showing 6 changed files with 224 additions and 103 deletions.
6 changes: 4 additions & 2 deletions tools/flakeguard/cmd/aggregate_results.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ var AggregateResultsCmd = &cobra.Command{

// Remove logs from test results for the report without logs
for i := range failedReportWithLogs.Results {
failedReportWithLogs.Results[i].Outputs = nil
failedReportWithLogs.Results[i].PassedOutputs = nil
failedReportWithLogs.Results[i].FailedOutputs = nil
failedReportWithLogs.Results[i].PackageOutputs = nil
}

Expand All @@ -137,7 +138,8 @@ var AggregateResultsCmd = &cobra.Command{

// Remove logs from test results for the aggregated report
for i := range aggregatedReport.Results {
aggregatedReport.Results[i].Outputs = nil
aggregatedReport.Results[i].PassedOutputs = nil
aggregatedReport.Results[i].FailedOutputs = nil
aggregatedReport.Results[i].PackageOutputs = nil
}

Expand Down
17 changes: 15 additions & 2 deletions tools/flakeguard/reports/data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ type TestResult struct {
Failures int
Successes int
Skips int
Outputs []string
Outputs map[string][]string `json:"-"` // Temporary storage for outputs during test run
PassedOutputs map[string][]string // Outputs for passed runs
FailedOutputs map[string][]string // Outputs for failed runs
Durations []time.Duration
PackageOutputs []string
TestPath string
Expand Down Expand Up @@ -191,7 +193,18 @@ func aggregateFromReports(reports ...*TestReport) (*TestReport, error) {
func mergeTestResults(a, b TestResult) TestResult {
a.Runs += b.Runs
a.Durations = append(a.Durations, b.Durations...)
a.Outputs = append(a.Outputs, b.Outputs...)
if a.PassedOutputs == nil {
a.PassedOutputs = make(map[string][]string)
}
if a.FailedOutputs == nil {
a.FailedOutputs = make(map[string][]string)
}
for runID, outputs := range b.PassedOutputs {
a.PassedOutputs[runID] = append(a.PassedOutputs[runID], outputs...)
}
for runID, outputs := range b.FailedOutputs {
a.FailedOutputs[runID] = append(a.FailedOutputs[runID], outputs...)
}
a.PackageOutputs = append(a.PackageOutputs, b.PackageOutputs...)
a.Successes += b.Successes
a.Failures += b.Failures
Expand Down
130 changes: 43 additions & 87 deletions tools/flakeguard/reports/data_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,11 +346,13 @@ func TestAggregateOutputs(t *testing.T) {
TestRunCount: 1,
Results: []TestResult{
{
TestName: "TestOutput",
TestPackage: "pkg1",
Runs: 1,
Successes: 1,
Outputs: []string{"Output from report1 test run"},
TestName: "TestOutput",
TestPackage: "pkg1",
Runs: 1,
Successes: 1,
PassedOutputs: map[string][]string{
"run1": {"Output from report1 test run"},
},
PackageOutputs: []string{"Package output from report1"},
},
},
Expand All @@ -361,11 +363,13 @@ func TestAggregateOutputs(t *testing.T) {
TestRunCount: 1,
Results: []TestResult{
{
TestName: "TestOutput",
TestPackage: "pkg1",
Runs: 1,
Successes: 1,
Outputs: []string{"Output from report2 test run"},
TestName: "TestOutput",
TestPackage: "pkg1",
Runs: 1,
Successes: 1,
PassedOutputs: map[string][]string{
"run2": {"Output from report2 test run"},
},
PackageOutputs: []string{"Package output from report2"},
},
},
Expand All @@ -382,18 +386,22 @@ func TestAggregateOutputs(t *testing.T) {

result := aggregatedReport.Results[0]

// Expected outputs
expectedOutputs := []string{
"Output from report1 test run",
"Output from report2 test run",
expectedOutputs := map[string][]string{
"run1": {
"Output from report1 test run",
},
"run2": {
"Output from report2 test run",
},
}

expectedPackageOutputs := []string{
"Package output from report1",
"Package output from report2",
}

if !reflect.DeepEqual(result.Outputs, expectedOutputs) {
t.Errorf("Expected Outputs %v, got %v", expectedOutputs, result.Outputs)
if !reflect.DeepEqual(result.PassedOutputs, expectedOutputs) {
t.Errorf("Expected Outputs %v, got %v", expectedOutputs, result.PassedOutputs)
}

if !reflect.DeepEqual(result.PackageOutputs, expectedPackageOutputs) {
Expand All @@ -407,11 +415,13 @@ func TestAggregateIdenticalOutputs(t *testing.T) {
TestRunCount: 1,
Results: []TestResult{
{
TestName: "TestIdenticalOutput",
TestPackage: "pkg1",
Runs: 1,
Successes: 1,
Outputs: []string{"Identical output"},
TestName: "TestIdenticalOutput",
TestPackage: "pkg1",
Runs: 1,
Successes: 1,
PassedOutputs: map[string][]string{
"run1": {"Identical output"},
},
PackageOutputs: []string{"Identical package output"},
},
},
Expand All @@ -422,11 +432,13 @@ func TestAggregateIdenticalOutputs(t *testing.T) {
TestRunCount: 1,
Results: []TestResult{
{
TestName: "TestIdenticalOutput",
TestPackage: "pkg1",
Runs: 1,
Successes: 1,
Outputs: []string{"Identical output"},
TestName: "TestIdenticalOutput",
TestPackage: "pkg1",
Runs: 1,
Successes: 1,
PassedOutputs: map[string][]string{
"run1": {"Identical output"},
},
PackageOutputs: []string{"Identical package output"},
},
},
Expand All @@ -443,80 +455,24 @@ func TestAggregateIdenticalOutputs(t *testing.T) {

result := aggregatedReport.Results[0]

// Expected outputs
expectedOutputs := []string{
"Identical output",
"Identical output",
expectedOutputs := map[string][]string{
"run1": {"Identical output", "Identical output"},
}

expectedPackageOutputs := []string{
"Identical package output",
"Identical package output",
}

if !reflect.DeepEqual(result.Outputs, expectedOutputs) {
t.Errorf("Expected Outputs %v, got %v", expectedOutputs, result.Outputs)
if !reflect.DeepEqual(result.PassedOutputs, expectedOutputs) {
t.Errorf("Expected Outputs %v, got %v", expectedOutputs, result.PassedOutputs)
}

if !reflect.DeepEqual(result.PackageOutputs, expectedPackageOutputs) {
t.Errorf("Expected PackageOutputs %v, got %v", expectedPackageOutputs, result.PackageOutputs)
}
}

// TestMergeTestResults tests the mergeTestResults function.
func TestMergeTestResults(t *testing.T) {
a := TestResult{
TestName: "TestA",
TestPackage: "pkg1",
Runs: 2,
Successes: 2,
Failures: 0,
Skips: 0,
Durations: []time.Duration{time.Second, time.Second},
Outputs: []string{"Output1", "Output2"},
PackageOutputs: []string{"PkgOutput1"},
Panic: false,
Race: false,
Skipped: false,
}

b := TestResult{
TestName: "TestA",
TestPackage: "pkg1",
Runs: 3,
Successes: 2,
Failures: 1,
Skips: 0,
Durations: []time.Duration{2 * time.Second, 2 * time.Second, 2 * time.Second},
Outputs: []string{"Output3", "Output4", "Output5"},
PackageOutputs: []string{"PkgOutput2"},
Panic: true,
Race: false,
Skipped: false,
}

merged := mergeTestResults(a, b)

expected := TestResult{
TestName: "TestA",
TestPackage: "pkg1",
Runs: 5,
Successes: 4,
Failures: 1,
Skips: 0,
Durations: []time.Duration{time.Second, time.Second, 2 * time.Second, 2 * time.Second, 2 * time.Second},
Outputs: []string{"Output1", "Output2", "Output3", "Output4", "Output5"},
PackageOutputs: []string{"PkgOutput1", "PkgOutput2"},
Panic: true,
Race: false,
Skipped: false,
PassRatio: 0.8,
}

if !reflect.DeepEqual(merged, expected) {
t.Errorf("Expected %+v, got %+v", expected, merged)
}
}

// TestAvgDuration tests the avgDuration function.
func TestAvgDuration(t *testing.T) {
durations := []time.Duration{
Expand Down
3 changes: 2 additions & 1 deletion tools/flakeguard/reports/io.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,8 @@ func SaveSummaryAsJSON(fs FileSystem, path string, summary SummaryData) error {
func SaveReportNoLogs(fs FileSystem, filePath string, report TestReport) error {
var filteredResults []TestResult
for _, r := range report.Results {
r.Outputs = nil
r.FailedOutputs = nil
r.PassedOutputs = nil
r.PackageOutputs = nil
filteredResults = append(filteredResults, r)
}
Expand Down
Loading

0 comments on commit ea4ffd8

Please sign in to comment.