Extendable metrics collection

cmd/eval-symflower-codegen-testing/cmd/evaluate.go

@@ -66,16 +66,18 @@ func (command *Evaluate) Execute(args []string) (err error) {
 	// Check that models and languages can be evaluated by executing the "plain" repositories.
 	log.Printf("Checking that models and languages can used for evaluation")
 	problemsPerModel := map[string][]error{}
+	metricsPerModel := map[string]evaluate.Metrics{}
 	for _, languageID := range command.Languages {
 		for _, modelID := range command.Models {
 			model := model.Models[modelID]
 			language := language.Languages[languageID]
 
-			ps, err := evaluate.EvaluateRepository(model, language, filepath.Join(command.TestdataPath, language.ID(), "plain"))
+			metrics, ps, err := evaluate.EvaluateRepository(model, language, filepath.Join(command.TestdataPath, language.ID(), "plain"))
 			problemsPerModel[modelID] = append(problemsPerModel[modelID], ps...)
 			if err != nil {
 				log.Fatalf("%+v", err)
 			}
+			metricsPerModel[model.ID()] = metricsPerModel[model.ID()].Add(metrics)
 		}
 	}
 
@@ -87,6 +89,8 @@ func (command *Evaluate) Execute(args []string) (err error) {
 				log.Printf("%+v:", p)
 			}
 		}
+
+		log.Printf("Evaluation score for %q: %s", modelID, metricsPerModel[modelID].StringPercentage())
 	}
 
 	return nil

evaluate/metrics.go

@@ -0,0 +1,31 @@
+package evaluate
+
+import (
+	"fmt"
+)
+
+// Metrics holds numerical benchmarking metrics.
+type Metrics struct {
+	// Total is the total number of benchmarking candidates.
+	Total uint
+	// Executed is the number of benchmarking candidates with successful execution.
+	Executed uint
+}
+
+// Add sums two metrics objects.
+func (m Metrics) Add(o Metrics) Metrics {
+	return Metrics{
+		Total:    m.Total + o.Total,
+		Executed: m.Executed + o.Total,
+	}
+}
+
+// String returns a string representation of the metrics.
+func (m Metrics) String() string {
+	return fmt.Sprintf("#executed=%d/%d", m.Executed, m.Total)
+}
+
+// String returns a string representation of the metrics.
+func (m Metrics) StringPercentage() string {
+	return fmt.Sprintf("#executed=%3.0f%%", float64(m.Executed)/float64(m.Total)*100.0)
+}

evaluate/repository.go

@@ -14,15 +14,15 @@ import (
 )
 
 // EvaluateRepository evaluate a repository with the given model and language.
-func EvaluateRepository(model model.Model, language language.Language, repositoryPath string) (problems []error, err error) {
+func EvaluateRepository(model model.Model, language language.Language, repositoryPath string) (metrics Metrics, problems []error, err error) {
 	log.Printf("Evaluating model %q using language %q and repository %q", model.ID(), language.ID(), repositoryPath)
 	defer func() {
 		log.Printf("Evaluated model %q using language %q and repository %q: encountered %d problems", model.ID(), language.ID(), repositoryPath, len(problems))
 	}()
 
 	temporaryPath, err := os.MkdirTemp("", "eval-symflower-codegen-testing")
 	if err != nil {
-		return problems, pkgerrors.WithStack(err)
+		return metrics, problems, pkgerrors.WithStack(err)
 	}
 	defer func() {
 		if e := os.RemoveAll(temporaryPath); e != nil {
@@ -35,15 +35,16 @@ func EvaluateRepository(model model.Model, language language.Language, repositor
 	}()
 	temporaryRepositoryPath := filepath.Join(temporaryPath, filepath.Base(repositoryPath))
 	if err := osutil.CopyTree(repositoryPath, temporaryRepositoryPath); err != nil {
-		return problems, pkgerrors.WithStack(err)
+		return metrics, problems, pkgerrors.WithStack(err)
 	}
 
 	filePaths, err := language.Files(repositoryPath)
 	if err != nil {
-		return problems, pkgerrors.WithStack(err)
+		return metrics, problems, pkgerrors.WithStack(err)
 	}
 
 	for _, filePath := range filePaths {
+		metrics.Total++
 		if err := model.GenerateTestsForFile(temporaryRepositoryPath, filePath); err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, filePath))
 
@@ -55,7 +56,8 @@ func EvaluateRepository(model model.Model, language language.Language, repositor
 
 			continue
 		}
+		metrics.Total++
 	}
 
-	return problems, nil
+	return metrics, problems, nil
 }