Forward logger objects to languages and models to forward their entri…

…es into the right log
symflower · Apr 18, 2024 · 249890b · 249890b
1 parent 3647eb9
commit 249890b
Show file tree

Hide file tree

Showing 15 changed files with 73 additions and 24 deletions.
diff --git a/evaluate/metrics/testing/assessments.go b/evaluate/metrics/testing/assessments.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
 )
 

diff --git a/evaluate/repository.go b/evaluate/repository.go
@@ -48,14 +48,14 @@ func EvaluateRepository(resultPath string, model model.Model, language language.
 		return nil, problems, pkgerrors.WithStack(err)
 	}
 
-	filePaths, err := language.Files(dataPath)
+	filePaths, err := language.Files(log, dataPath)
 	if err != nil {
 		return nil, problems, pkgerrors.WithStack(err)
 	}
 
 	repositoryAssessment = metrics.NewAssessments()
 	for _, filePath := range filePaths {
-		assessments, err := model.GenerateTestsForFile(language, temporaryRepositoryPath, filePath)
+		assessments, err := model.GenerateTestsForFile(log, language, temporaryRepositoryPath, filePath)
 		if err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, filePath))
 
@@ -64,7 +64,7 @@ func EvaluateRepository(resultPath string, model model.Model, language language.
 		repositoryAssessment.Add(assessments)
 		repositoryAssessment[metrics.AssessmentKeyResponseNoError]++
 
-		coverage, err := language.Execute(temporaryRepositoryPath)
+		coverage, err := language.Execute(log, temporaryRepositoryPath)
 		if err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, filePath))
 

diff --git a/language/golang.go b/language/golang.go
@@ -2,6 +2,7 @@ package language
 
 import (
 	"errors"
+	"log"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -34,7 +35,7 @@ func (language *LanguageGolang) Name() (id string) {
 }
 
 // Files returns a list of relative file paths of the repository that should be evaluated.
-func (language *LanguageGolang) Files(repositoryPath string) (filePaths []string, err error) {
+func (language *LanguageGolang) Files(log *log.Logger, repositoryPath string) (filePaths []string, err error) {
 	repositoryPath, err = filepath.Abs(repositoryPath)
 	if err != nil {
 		return nil, pkgerrors.WithStack(err)
@@ -62,8 +63,8 @@ var languageGoCoverageMatch = regexp.MustCompile(`(?m)^coverage: (\d+\.?\d+)% of
 var languageGoNoCoverageMatch = regexp.MustCompile(`(?m)^coverage: \[no statements\]$`)
 
 // Execute invokes the language specific testing on the given repository.
-func (language *LanguageGolang) Execute(repositoryPath string) (coverage float64, err error) {
-	stdout, _, err := util.CommandWithResult(&util.Command{
+func (language *LanguageGolang) Execute(log *log.Logger, repositoryPath string) (coverage float64, err error) {
+	stdout, _, err := util.CommandWithResult(log, &util.Command{
 		Command: []string{
 			"gotestsum",
 			"--format", "standard-verbose", // Keep formatting consistent.

diff --git a/language/golang_test.go b/language/golang_test.go
@@ -9,6 +9,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/zimmski/osutil"
 	"github.com/zimmski/osutil/bytesutil"
+
+	"github.com/symflower/eval-dev-quality/log"
 )
 
 func TestLanguageGolangFiles(t *testing.T) {
@@ -25,10 +27,17 @@ func TestLanguageGolangFiles(t *testing.T) {
 
 	validate := func(t *testing.T, tc *testCase) {
 		t.Run(tc.Name, func(t *testing.T) {
+			log, logger := log.Buffer()
+			defer func() {
+				if t.Failed() {
+					t.Log(log.String())
+				}
+			}()
+
 			if tc.LanguageGolang == nil {
 				tc.LanguageGolang = &LanguageGolang{}
 			}
-			actualFilePaths, actualError := tc.LanguageGolang.Files(tc.RepositoryPath)
+			actualFilePaths, actualError := tc.LanguageGolang.Files(logger, tc.RepositoryPath)
 
 			assert.Equal(t, tc.ExpectedFilePaths, actualFilePaths)
 			assert.Equal(t, tc.ExpectedError, actualError)
@@ -62,6 +71,13 @@ func TestLanguageGolangExecute(t *testing.T) {
 
 	validate := func(t *testing.T, tc *testCase) {
 		t.Run(tc.Name, func(t *testing.T) {
+			log, logger := log.Buffer()
+			defer func() {
+				if t.Failed() {
+					t.Log(log.String())
+				}
+			}()
+
 			temporaryPath := t.TempDir()
 			repositoryPath := filepath.Join(temporaryPath, filepath.Base(tc.RepositoryPath))
 			require.NoError(t, osutil.CopyTree(tc.RepositoryPath, repositoryPath))
@@ -73,7 +89,7 @@ func TestLanguageGolangExecute(t *testing.T) {
 			if tc.LanguageGolang == nil {
 				tc.LanguageGolang = &LanguageGolang{}
 			}
-			actualCoverage, actualError := tc.LanguageGolang.Execute(repositoryPath)
+			actualCoverage, actualError := tc.LanguageGolang.Execute(logger, repositoryPath)
 
 			if tc.ExpectedError != nil {
 				assert.ErrorIs(t, actualError, tc.ExpectedError)

diff --git a/language/language.go b/language/language.go
@@ -1,6 +1,8 @@
 package language
 
 import (
+	"log"
+
 	pkgerrors "github.com/pkg/errors"
 )
 
@@ -12,10 +14,10 @@ type Language interface {
 	Name() (id string)
 
 	// Files returns a list of relative file paths of the repository that should be evaluated.
-	Files(repositoryPath string) (filePaths []string, err error)
+	Files(log *log.Logger, repositoryPath string) (filePaths []string, err error)
 
 	// Execute invokes the language specific testing on the given repository.
-	Execute(repositoryPath string) (coverage float64, err error)
+	Execute(log *log.Logger, repositoryPath string) (coverage float64, err error)
 }
 
 // Languages holds a register of all languages.

diff --git a/log/logger.go b/log/logger.go
@@ -1,6 +1,7 @@
 package log
 
 import (
+	"bytes"
 	"io"
 	"log"
 	"os"
@@ -9,6 +10,14 @@ import (
 	pkgerrors "github.com/pkg/errors"
 )
 
+// Buffer returns a logger that writes to a buffer.
+func Buffer() (buffer *bytes.Buffer, logger *log.Logger) {
+	buffer = new(bytes.Buffer)
+	logger = log.New(buffer, "", log.LstdFlags)
+
+	return buffer, logger
+}
+
 // File returns a logger that writes to a file.
 func File(path string) (logger *log.Logger, loggerClose func(), err error) {
 	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {

diff --git a/model/llm/llm.go b/model/llm/llm.go
@@ -78,7 +78,7 @@ func (m *llm) ID() (id string) {
 }
 
 // GenerateTestsForFile generates test files for the given implementation file in a repository.
-func (m *llm) GenerateTestsForFile(language language.Language, repositoryPath string, filePath string) (assessment metrics.Assessments, err error) {
+func (m *llm) GenerateTestsForFile(log *log.Logger, language language.Language, repositoryPath string, filePath string) (assessment metrics.Assessments, err error) {
 	data, err := os.ReadFile(filepath.Join(repositoryPath, filePath))
 	if err != nil {
 		return nil, pkgerrors.WithStack(err)

diff --git a/model/llm/llm_test.go b/model/llm/llm_test.go
@@ -12,9 +12,9 @@ import (
 	"github.com/zimmski/osutil/bytesutil"
 
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
-	"github.com/symflower/eval-dev-quality/language"
-
 	metricstesting "github.com/symflower/eval-dev-quality/evaluate/metrics/testing"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/log"
 	providertesting "github.com/symflower/eval-dev-quality/provider/testing"
 )
 
@@ -36,6 +36,13 @@ func TestModelLLMGenerateTestsForFile(t *testing.T) {
 
 	validate := func(t *testing.T, tc *testCase) {
 		t.Run(tc.Name, func(t *testing.T) {
+			log, logger := log.Buffer()
+			defer func() {
+				if t.Failed() {
+					t.Log(log.String())
+				}
+			}()
+
 			temporaryPath := t.TempDir()
 			temporaryPath = filepath.Join(temporaryPath, "native")
 			require.NoError(t, os.Mkdir(temporaryPath, 0755))
@@ -46,7 +53,7 @@ func TestModelLLMGenerateTestsForFile(t *testing.T) {
 			tc.SetupMock(mock)
 			llm := NewLLMModel(mock, tc.ModelID)
 
-			actualAssessment, actualError := llm.GenerateTestsForFile(tc.Language, temporaryPath, tc.SourceFilePath)
+			actualAssessment, actualError := llm.GenerateTestsForFile(logger, tc.Language, temporaryPath, tc.SourceFilePath)
 			assert.NoError(t, actualError)
 			metricstesting.AssertAssessmentsEqual(t, tc.ExpectedAssessment, actualAssessment)
 

diff --git a/model/llm/prompt/parse.go b/model/llm/prompt/parse.go
@@ -4,8 +4,9 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/symflower/eval-dev-quality/evaluate/metrics"
 	"github.com/zimmski/osutil/bytesutil"
+
+	"github.com/symflower/eval-dev-quality/evaluate/metrics"
 )
 
 var (

diff --git a/model/llm/prompt/parse_test.go b/model/llm/prompt/parse_test.go
@@ -5,9 +5,9 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/symflower/eval-dev-quality/evaluate/metrics"
 	"github.com/zimmski/osutil/bytesutil"
 
+	"github.com/symflower/eval-dev-quality/evaluate/metrics"
 	metricstesting "github.com/symflower/eval-dev-quality/evaluate/metrics/testing"
 )
 

diff --git a/model/model.go b/model/model.go
@@ -1,6 +1,8 @@
 package model
 
 import (
+	"log"
+
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
 	"github.com/symflower/eval-dev-quality/language"
 )
@@ -11,5 +13,5 @@ type Model interface {
 	ID() (id string)
 
 	// GenerateTestsForFile generates test files for the given implementation file in a repository.
-	GenerateTestsForFile(language language.Language, repositoryPath string, filePath string) (assessments metrics.Assessments, err error)
+	GenerateTestsForFile(log *log.Logger, language language.Language, repositoryPath string, filePath string) (assessments metrics.Assessments, err error)
 }
diff --git a/model/symflower/symflower.go b/model/symflower/symflower.go
@@ -1,6 +1,8 @@
 package symflower
 
 import (
+	"log"
+
 	pkgerrors "github.com/pkg/errors"
 
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
@@ -21,8 +23,8 @@ func (m *ModelSymflower) ID() (id string) {
 }
 
 // GenerateTestsForFile generates test files for the given implementation file in a repository.
-func (m *ModelSymflower) GenerateTestsForFile(language language.Language, repositoryPath string, filePath string) (assessment metrics.Assessments, err error) {
-	_, _, err = util.CommandWithResult(&util.Command{
+func (m *ModelSymflower) GenerateTestsForFile(log *log.Logger, language language.Language, repositoryPath string, filePath string) (assessment metrics.Assessments, err error) {
+	_, _, err = util.CommandWithResult(log, &util.Command{
 		Command: []string{
 			"symflower", "unit-tests",
 			"--workspace", repositoryPath,

diff --git a/model/symflower/symflower_test.go b/model/symflower/symflower_test.go
@@ -9,9 +9,9 @@ import (
 	"github.com/zimmski/osutil"
 
 	"github.com/symflower/eval-dev-quality/evaluate/metrics"
-	"github.com/symflower/eval-dev-quality/language"
-
 	metricstesting "github.com/symflower/eval-dev-quality/evaluate/metrics/testing"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/log"
 )
 
 func TestModelSymflowerGenerateTestsForFile(t *testing.T) {
@@ -33,6 +33,13 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) {
 
 	validate := func(t *testing.T, tc *testCase) {
 		t.Run(tc.Name, func(t *testing.T) {
+			log, logger := log.Buffer()
+			defer func() {
+				if t.Failed() {
+					t.Log(log.String())
+				}
+			}()
+
 			temporaryPath := t.TempDir()
 			repositoryPath := filepath.Join(temporaryPath, filepath.Base(tc.RepositoryPath))
 			require.NoError(t, osutil.CopyTree(tc.RepositoryPath, repositoryPath))
@@ -44,7 +51,7 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) {
 			if tc.ModelSymflower == nil {
 				tc.ModelSymflower = &ModelSymflower{}
 			}
-			actualAssessment, actualError := tc.ModelSymflower.GenerateTestsForFile(tc.Language, repositoryPath, tc.FilePath)
+			actualAssessment, actualError := tc.ModelSymflower.GenerateTestsForFile(logger, tc.Language, repositoryPath, tc.FilePath)
 
 			if tc.ExpectedError != nil {
 				assert.ErrorIs(t, tc.ExpectedError, actualError)
@@ -53,7 +60,7 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) {
 			}
 			metricstesting.AssertAssessmentsEqual(t, tc.ExpectedAssessment, actualAssessment)
 
-			actualCoverage, err := tc.Language.Execute(repositoryPath)
+			actualCoverage, err := tc.Language.Execute(logger, repositoryPath)
 			require.NoError(t, err)
 			assert.Equal(t, tc.ExpectedCoverage, actualCoverage)
 		})

diff --git a/provider/provider.go b/provider/provider.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	pkgerrors "github.com/pkg/errors"
+
 	"github.com/symflower/eval-dev-quality/model"
 )
 

diff --git a/util/exec.go b/util/exec.go
@@ -21,7 +21,7 @@ type Command struct {
 }
 
 // CommandWithResult executes a command, and prints and returns STDERR/STDOUT.
-func CommandWithResult(command *Command) (stdout string, stderr string, err error) {
+func CommandWithResult(log *log.Logger, command *Command) (stdout string, stderr string, err error) {
 	log.Printf("$ %s", strings.Join(command.Command, " "))
 
 	var stdoutWriter bytes.Buffer
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,6 +4,7 @@ import ( @@
     	"testing"
     	"github.com/stretchr/testify/assert"
     	"github.com/symflower/eval-dev-quality/evaluate/metrics"
     )
@@ Expand Down @@