Extract statement coverage when executing repositories

evaluate/metrics.go

@@ -8,6 +8,7 @@ import (
 
 	pkgerrors "github.com/pkg/errors"
 	"golang.org/x/exp/maps"
+	"gonum.org/v1/gonum/stat"
 )
 
 // Metrics holds numerical benchmarking metrics.
@@ -16,31 +17,37 @@ type Metrics struct {
 	Total uint
 	// Executed is the number of benchmarking candidates with successful execution.
 	Executed uint
+
+	// Coverage holds the coverage of the benchmarking candidates.
+	Coverage []float64
 }
 
 // Add sums two metrics objects.
 func (m Metrics) Add(o Metrics) Metrics {
 	return Metrics{
 		Total:    m.Total + o.Total,
 		Executed: m.Executed + o.Total,
+
+		Coverage: append(m.Coverage, o.Coverage...),
 	}
 }
 
 // String returns a string representation of the metrics.
 func (m Metrics) String() string {
-	return fmt.Sprintf("#executed=%d/%d", m.Executed, m.Total)
+	return fmt.Sprintf("#executed=%d/%d, coverage=%v", m.Executed, m.Total, m.Coverage)
 }
 
 // String returns a string representation of the metrics.
 func (m Metrics) StringPercentage() string {
-	return fmt.Sprintf("#executed=%3.0f%%", float64(m.Executed)/float64(m.Total)*100.0)
+	return fmt.Sprintf("#executed=%3.0f%%, average coverage=%3.0f%%", float64(m.Executed)/float64(m.Total)*100.0, stat.Mean(m.Coverage, nil))
 }
 
 // StringCSV returns a CSV row string representation of the metrics.
 func (m Metrics) StringCSV() []string {
 	return []string{
 		fmt.Sprintf("%d", m.Total),
 		fmt.Sprintf("%d", m.Executed),
+		fmt.Sprintf("%.0f", stat.Mean(m.Coverage, nil)),
 	}
 }
 
@@ -49,7 +56,7 @@ func FormatStringCSV(metricsPerModel map[string]Metrics) (string, error) {
 	var out strings.Builder
 	csv := csv.NewWriter(&out)
 
-	if err := csv.Write([]string{"model", "total", "executed"}); err != nil {
+	if err := csv.Write([]string{"model", "total", "executed", "coverage"}); err != nil {
 		return "", err
 	}
 	categories := maps.Keys(metricsPerModel)

evaluate/metrics_test.go

@@ -33,7 +33,7 @@ func TestFormatStringCSV(t *testing.T) {
 		},
 
 		ExpectedString: `
-			model,total,executed
+			model,total,executed,coverage
 			Model,0,0,NaN
 		`,
 	})
@@ -44,17 +44,19 @@ func TestFormatStringCSV(t *testing.T) {
 			"ModelA": Metrics{
 				Total:    5,
 				Executed: 3,
+				Coverage: []float64{100.0},
 			},
 			"ModelB": Metrics{
 				Total:    4,
 				Executed: 2,
+				Coverage: []float64{70.0},
 			},
 		},
 
 		ExpectedString: `
-			model,total,executed
-			ModelA,5,3
-			ModelB,4,2
+			model,total,executed,coverage
+			ModelA,5,3,100
+			ModelB,4,2,70
 		`,
 	})
 }

evaluate/repository.go

@@ -3,6 +3,7 @@ package evaluate
 import (
 	"errors"
 	"log"
+	"math"
 	"os"
 	"path/filepath"
 
@@ -51,12 +52,16 @@ func EvaluateRepository(model model.Model, language language.Language, repositor
 			continue
 		}
 
-		if err := language.Execute(temporaryRepositoryPath); err != nil {
+		coverage, err := language.Execute(temporaryRepositoryPath)
+		if err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, filePath))
 
 			continue
 		}
 		metrics.Total++
+		if !math.IsNaN(coverage) {
+			metrics.Coverage = append(metrics.Coverage, coverage)
+		}
 	}
 
 	return metrics, problems, nil

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/zimmski/osutil v1.1.1-0.20240329123339-0f31a74c026a
 	golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8
+	gonum.org/v1/gonum v0.15.0
 )
 
 require (

go.sum

@@ -43,6 +43,8 @@ golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
 golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ=
+gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

language/golang.go

@@ -2,6 +2,7 @@ package language
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -53,15 +54,18 @@ func (language *LanguageGolang) Files(repositoryPath string) (filePaths []string
 }
 
 var languageGoNoTestsMatch = regexp.MustCompile(`(?m)^DONE (\d+) tests.*in (.+?)$`)
+var languageGoCoverageMatch = regexp.MustCompile(`(?m)^coverage: (\d+\.?\d+)% of statements`)
+var languageGoNoCoverageMatch = regexp.MustCompile(`(?m)^coverage: \[no statements\]$`)
 
 // Execute invokes the language specific testing on the given repository.
-func (language *LanguageGolang) Execute(repositoryPath string) (err error) {
+func (language *LanguageGolang) Execute(repositoryPath string) (coverage float64, err error) {
 	stdout, _, err := util.CommandWithResult(&util.Command{
 		Command: []string{
 			"gotestsum",
 			"--format", "standard-verbose", // Keep formatting consistent.
 			"--hide-summary", "skipped", // We are not interested in skipped tests, because they are the same as no tests at all.
 			"--",       // Let the real Go "test" tool options begin.
+			"-cover",   // Enable statement coverage.
 			"-v",       // Output with the maximum information for easier debugging.
 			"-vet=off", // Disable all linter checks, because those should be part of a different task.
 			"./...",    // Always execute all tests of the repository in case multiple test files have been generated.
@@ -70,19 +74,33 @@ func (language *LanguageGolang) Execute(repositoryPath string) (err error) {
 		Directory: repositoryPath,
 	})
 	if err != nil {
-		return pkgerrors.WithStack(err)
+		return 0.0, pkgerrors.WithStack(err)
 	}
 
 	ms := languageGoNoTestsMatch.FindStringSubmatch(stdout)
 	if ms == nil {
-		return pkgerrors.WithStack(errors.New("could not find Go test summary"))
+		return 0.0, pkgerrors.WithStack(errors.New("could not find Go test summary"))
 	}
 	testCount, err := strconv.ParseUint(ms[1], 10, 64)
 	if err != nil {
-		return pkgerrors.WithStack(err)
+		return 0.0, pkgerrors.WithStack(err)
 	} else if testCount == 0 {
-		return pkgerrors.WithStack(ErrNoTestFound)
+		return 0.0, pkgerrors.WithStack(ErrNoTestFound)
 	}
 
-	return nil
+	if languageGoNoCoverageMatch.MatchString(stdout) {
+		return 0.0, nil
+	}
+
+	mc := languageGoCoverageMatch.FindStringSubmatch(stdout)
+	if mc == nil {
+		fmt.Println(mc, stdout)
+		return 0.0, pkgerrors.WithStack(errors.New("could not find Go coverage report"))
+	}
+	coverage, err = strconv.ParseFloat(mc[1], 64)
+	if err != nil {
+		return 0.0, pkgerrors.WithStack(err)
+	}
+
+	return coverage, nil
 }

language/golang_test.go

@@ -55,6 +55,7 @@ func TestLanguageGolangExecute(t *testing.T) {
 		RepositoryPath   string
 		RepositoryChange func(t *testing.T, repositoryPath string)
 
+		ExpectedCoverage  float64
 		ExpectedError     error
 		ExpectedErrorText string
 	}
@@ -72,14 +73,15 @@ func TestLanguageGolangExecute(t *testing.T) {
 			if tc.LanguageGolang == nil {
 				tc.LanguageGolang = &LanguageGolang{}
 			}
-			actualError := tc.LanguageGolang.Execute(repositoryPath)
+			actualCoverage, actualError := tc.LanguageGolang.Execute(repositoryPath)
 
 			if tc.ExpectedError != nil {
 				assert.ErrorIs(t, actualError, tc.ExpectedError)
 			} else if actualError != nil && tc.ExpectedErrorText != "" {
 				assert.ErrorContains(t, actualError, tc.ExpectedErrorText)
 			} else {
 				assert.NoError(t, actualError)
+				assert.Equal(t, tc.ExpectedCoverage, actualCoverage)
 			}
 		})
 	}
@@ -110,6 +112,8 @@ func TestLanguageGolangExecute(t *testing.T) {
 					}
 				`)), 0660))
 			},
+
+			ExpectedCoverage: 100,
 		})
 
 		validate(t, &testCase{

language/language.go

@@ -13,7 +13,7 @@ type Language interface {
 	Files(repositoryPath string) (filePaths []string, err error)
 
 	// Execute invokes the language specific testing on the given repository.
-	Execute(repositoryPath string) (err error)
+	Execute(repositoryPath string) (coverage float64, err error)
 }
 
 // Languages holds a register of all languages.

model/symflower/symflower_test.go

@@ -22,6 +22,7 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) {
 		RepositoryChange func(t *testing.T, repositoryPath string)
 		FilePath         string
 
+		ExpectedCoverage  float64
 		ExpectedError     error
 		ExpectedErrorText string
 	}
@@ -47,7 +48,9 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) {
 				assert.ErrorContains(t, actualErr, tc.ExpectedErrorText)
 			}
 
-			require.NoError(t, tc.Language.Execute(repositoryPath))
+			actualCoverage, err := tc.Language.Execute(repositoryPath)
+			require.NoError(t, err)
+			assert.Equal(t, tc.ExpectedCoverage, actualCoverage)
 		})
 	}
 
@@ -58,5 +61,7 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) {
 
 		RepositoryPath: "../../testdata/golang/plain/",
 		FilePath:       "plain.go",
+
+		ExpectedCoverage: 100,
 	})
 }

testdata/golang/plain/plain.go

@@ -1,4 +1,5 @@
 package native
 
 func plain() {
+	return // This does not do anything but it gives us a line to cover.
 }