From 75293e26f1566df491fb420b2365c6f735ffe80b Mon Sep 17 00:00:00 2001 From: Markus Zimmermann Date: Thu, 18 Apr 2024 15:58:08 +0200 Subject: [PATCH] Forward logger objects to languages and models to forward their entries into the right log --- evaluate/metrics/testing/assessments.go | 1 + evaluate/repository.go | 6 +++--- language/golang.go | 7 ++++--- language/golang_test.go | 20 ++++++++++++++++++-- language/language.go | 6 ++++-- log/logger.go | 9 +++++++++ model/llm/llm.go | 2 +- model/llm/llm_test.go | 13 ++++++++++--- model/llm/prompt/parse.go | 3 ++- model/llm/prompt/parse_test.go | 2 +- model/model.go | 4 +++- model/symflower/symflower.go | 6 ++++-- model/symflower/symflower_test.go | 15 +++++++++++---- provider/provider.go | 1 + util/exec.go | 2 +- 15 files changed, 73 insertions(+), 24 deletions(-) diff --git a/evaluate/metrics/testing/assessments.go b/evaluate/metrics/testing/assessments.go index 1fa163693..7e36e257c 100644 --- a/evaluate/metrics/testing/assessments.go +++ b/evaluate/metrics/testing/assessments.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/symflower/eval-dev-quality/evaluate/metrics" ) diff --git a/evaluate/repository.go b/evaluate/repository.go index cc97685ae..a3c069bc3 100644 --- a/evaluate/repository.go +++ b/evaluate/repository.go @@ -48,14 +48,14 @@ func EvaluateRepository(resultPath string, model model.Model, language language. return nil, problems, pkgerrors.WithStack(err) } - filePaths, err := language.Files(dataPath) + filePaths, err := language.Files(log, dataPath) if err != nil { return nil, problems, pkgerrors.WithStack(err) } repositoryAssessment = metrics.NewAssessments() for _, filePath := range filePaths { - assessments, err := model.GenerateTestsForFile(language, temporaryRepositoryPath, filePath) + assessments, err := model.GenerateTestsForFile(log, language, temporaryRepositoryPath, filePath) if err != nil { problems = append(problems, pkgerrors.WithMessage(err, filePath)) @@ -64,7 +64,7 @@ func EvaluateRepository(resultPath string, model model.Model, language language. repositoryAssessment.Add(assessments) repositoryAssessment[metrics.AssessmentKeyResponseNoError]++ - coverage, err := language.Execute(temporaryRepositoryPath) + coverage, err := language.Execute(log, temporaryRepositoryPath) if err != nil { problems = append(problems, pkgerrors.WithMessage(err, filePath)) diff --git a/language/golang.go b/language/golang.go index 23f30998a..9854cbdfc 100644 --- a/language/golang.go +++ b/language/golang.go @@ -2,6 +2,7 @@ package language import ( "errors" + "log" "os" "path/filepath" "regexp" @@ -34,7 +35,7 @@ func (language *LanguageGolang) Name() (id string) { } // Files returns a list of relative file paths of the repository that should be evaluated. -func (language *LanguageGolang) Files(repositoryPath string) (filePaths []string, err error) { +func (language *LanguageGolang) Files(log *log.Logger, repositoryPath string) (filePaths []string, err error) { repositoryPath, err = filepath.Abs(repositoryPath) if err != nil { return nil, pkgerrors.WithStack(err) @@ -62,8 +63,8 @@ var languageGoCoverageMatch = regexp.MustCompile(`(?m)^coverage: (\d+\.?\d+)% of var languageGoNoCoverageMatch = regexp.MustCompile(`(?m)^coverage: \[no statements\]$`) // Execute invokes the language specific testing on the given repository. -func (language *LanguageGolang) Execute(repositoryPath string) (coverage float64, err error) { - stdout, _, err := util.CommandWithResult(&util.Command{ +func (language *LanguageGolang) Execute(log *log.Logger, repositoryPath string) (coverage float64, err error) { + stdout, _, err := util.CommandWithResult(log, &util.Command{ Command: []string{ "gotestsum", "--format", "standard-verbose", // Keep formatting consistent. diff --git a/language/golang_test.go b/language/golang_test.go index dd0d6212a..691cbe94d 100644 --- a/language/golang_test.go +++ b/language/golang_test.go @@ -9,6 +9,8 @@ import ( "github.com/stretchr/testify/require" "github.com/zimmski/osutil" "github.com/zimmski/osutil/bytesutil" + + "github.com/symflower/eval-dev-quality/log" ) func TestLanguageGolangFiles(t *testing.T) { @@ -25,10 +27,17 @@ func TestLanguageGolangFiles(t *testing.T) { validate := func(t *testing.T, tc *testCase) { t.Run(tc.Name, func(t *testing.T) { + log, logger := log.Buffer() + defer func() { + if t.Failed() { + assert.Empty(t, log.String()) + } + }() + if tc.LanguageGolang == nil { tc.LanguageGolang = &LanguageGolang{} } - actualFilePaths, actualError := tc.LanguageGolang.Files(tc.RepositoryPath) + actualFilePaths, actualError := tc.LanguageGolang.Files(logger, tc.RepositoryPath) assert.Equal(t, tc.ExpectedFilePaths, actualFilePaths) assert.Equal(t, tc.ExpectedError, actualError) @@ -62,6 +71,13 @@ func TestLanguageGolangExecute(t *testing.T) { validate := func(t *testing.T, tc *testCase) { t.Run(tc.Name, func(t *testing.T) { + log, logger := log.Buffer() + defer func() { + if t.Failed() { + assert.Empty(t, log.String()) + } + }() + temporaryPath := t.TempDir() repositoryPath := filepath.Join(temporaryPath, filepath.Base(tc.RepositoryPath)) require.NoError(t, osutil.CopyTree(tc.RepositoryPath, repositoryPath)) @@ -73,7 +89,7 @@ func TestLanguageGolangExecute(t *testing.T) { if tc.LanguageGolang == nil { tc.LanguageGolang = &LanguageGolang{} } - actualCoverage, actualError := tc.LanguageGolang.Execute(repositoryPath) + actualCoverage, actualError := tc.LanguageGolang.Execute(logger, repositoryPath) if tc.ExpectedError != nil { assert.ErrorIs(t, actualError, tc.ExpectedError) diff --git a/language/language.go b/language/language.go index 0cca25834..f1549442d 100644 --- a/language/language.go +++ b/language/language.go @@ -1,6 +1,8 @@ package language import ( + "log" + pkgerrors "github.com/pkg/errors" ) @@ -12,10 +14,10 @@ type Language interface { Name() (id string) // Files returns a list of relative file paths of the repository that should be evaluated. - Files(repositoryPath string) (filePaths []string, err error) + Files(log *log.Logger, repositoryPath string) (filePaths []string, err error) // Execute invokes the language specific testing on the given repository. - Execute(repositoryPath string) (coverage float64, err error) + Execute(log *log.Logger, repositoryPath string) (coverage float64, err error) } // Languages holds a register of all languages. diff --git a/log/logger.go b/log/logger.go index 4179b1581..c9dde8dc5 100644 --- a/log/logger.go +++ b/log/logger.go @@ -1,6 +1,7 @@ package log import ( + "bytes" "io" "log" "os" @@ -9,6 +10,14 @@ import ( pkgerrors "github.com/pkg/errors" ) +// Buffer returns a logger that writes to a buffer. +func Buffer() (buffer *bytes.Buffer, logger *log.Logger) { + buffer = new(bytes.Buffer) + logger = log.New(buffer, "", log.LstdFlags) + + return buffer, logger +} + // File returns a logger that writes to a file. func File(path string) (logger *log.Logger, loggerClose func(), err error) { if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { diff --git a/model/llm/llm.go b/model/llm/llm.go index 70b8d67f4..94f2b4f64 100644 --- a/model/llm/llm.go +++ b/model/llm/llm.go @@ -78,7 +78,7 @@ func (m *llm) ID() (id string) { } // GenerateTestsForFile generates test files for the given implementation file in a repository. -func (m *llm) GenerateTestsForFile(language language.Language, repositoryPath string, filePath string) (assessment metrics.Assessments, err error) { +func (m *llm) GenerateTestsForFile(log *log.Logger, language language.Language, repositoryPath string, filePath string) (assessment metrics.Assessments, err error) { data, err := os.ReadFile(filepath.Join(repositoryPath, filePath)) if err != nil { return nil, pkgerrors.WithStack(err) diff --git a/model/llm/llm_test.go b/model/llm/llm_test.go index 5085e8246..cad179c06 100644 --- a/model/llm/llm_test.go +++ b/model/llm/llm_test.go @@ -12,9 +12,9 @@ import ( "github.com/zimmski/osutil/bytesutil" "github.com/symflower/eval-dev-quality/evaluate/metrics" - "github.com/symflower/eval-dev-quality/language" - metricstesting "github.com/symflower/eval-dev-quality/evaluate/metrics/testing" + "github.com/symflower/eval-dev-quality/language" + "github.com/symflower/eval-dev-quality/log" providertesting "github.com/symflower/eval-dev-quality/provider/testing" ) @@ -36,6 +36,13 @@ func TestModelLLMGenerateTestsForFile(t *testing.T) { validate := func(t *testing.T, tc *testCase) { t.Run(tc.Name, func(t *testing.T) { + log, logger := log.Buffer() + defer func() { + if t.Failed() { + assert.Empty(t, log.String()) + } + }() + temporaryPath := t.TempDir() temporaryPath = filepath.Join(temporaryPath, "native") require.NoError(t, os.Mkdir(temporaryPath, 0755)) @@ -46,7 +53,7 @@ func TestModelLLMGenerateTestsForFile(t *testing.T) { tc.SetupMock(mock) llm := NewLLMModel(mock, tc.ModelID) - actualAssessment, actualError := llm.GenerateTestsForFile(tc.Language, temporaryPath, tc.SourceFilePath) + actualAssessment, actualError := llm.GenerateTestsForFile(logger, tc.Language, temporaryPath, tc.SourceFilePath) assert.NoError(t, actualError) metricstesting.AssertAssessmentsEqual(t, tc.ExpectedAssessment, actualAssessment) diff --git a/model/llm/prompt/parse.go b/model/llm/prompt/parse.go index 852743798..afbb8501b 100644 --- a/model/llm/prompt/parse.go +++ b/model/llm/prompt/parse.go @@ -4,8 +4,9 @@ import ( "regexp" "strings" - "github.com/symflower/eval-dev-quality/evaluate/metrics" "github.com/zimmski/osutil/bytesutil" + + "github.com/symflower/eval-dev-quality/evaluate/metrics" ) var ( diff --git a/model/llm/prompt/parse_test.go b/model/llm/prompt/parse_test.go index fae61ff12..1ee99af51 100644 --- a/model/llm/prompt/parse_test.go +++ b/model/llm/prompt/parse_test.go @@ -5,9 +5,9 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/symflower/eval-dev-quality/evaluate/metrics" "github.com/zimmski/osutil/bytesutil" + "github.com/symflower/eval-dev-quality/evaluate/metrics" metricstesting "github.com/symflower/eval-dev-quality/evaluate/metrics/testing" ) diff --git a/model/model.go b/model/model.go index 01210a723..c159a6e48 100644 --- a/model/model.go +++ b/model/model.go @@ -1,6 +1,8 @@ package model import ( + "log" + "github.com/symflower/eval-dev-quality/evaluate/metrics" "github.com/symflower/eval-dev-quality/language" ) @@ -11,5 +13,5 @@ type Model interface { ID() (id string) // GenerateTestsForFile generates test files for the given implementation file in a repository. - GenerateTestsForFile(language language.Language, repositoryPath string, filePath string) (assessments metrics.Assessments, err error) + GenerateTestsForFile(log *log.Logger, language language.Language, repositoryPath string, filePath string) (assessments metrics.Assessments, err error) } diff --git a/model/symflower/symflower.go b/model/symflower/symflower.go index eb706d1e0..c04044bf3 100644 --- a/model/symflower/symflower.go +++ b/model/symflower/symflower.go @@ -1,6 +1,8 @@ package symflower import ( + "log" + pkgerrors "github.com/pkg/errors" "github.com/symflower/eval-dev-quality/evaluate/metrics" @@ -21,8 +23,8 @@ func (m *ModelSymflower) ID() (id string) { } // GenerateTestsForFile generates test files for the given implementation file in a repository. -func (m *ModelSymflower) GenerateTestsForFile(language language.Language, repositoryPath string, filePath string) (assessment metrics.Assessments, err error) { - _, _, err = util.CommandWithResult(&util.Command{ +func (m *ModelSymflower) GenerateTestsForFile(log *log.Logger, language language.Language, repositoryPath string, filePath string) (assessment metrics.Assessments, err error) { + _, _, err = util.CommandWithResult(log, &util.Command{ Command: []string{ "symflower", "unit-tests", "--workspace", repositoryPath, diff --git a/model/symflower/symflower_test.go b/model/symflower/symflower_test.go index e3477e188..b808308ac 100644 --- a/model/symflower/symflower_test.go +++ b/model/symflower/symflower_test.go @@ -9,9 +9,9 @@ import ( "github.com/zimmski/osutil" "github.com/symflower/eval-dev-quality/evaluate/metrics" - "github.com/symflower/eval-dev-quality/language" - metricstesting "github.com/symflower/eval-dev-quality/evaluate/metrics/testing" + "github.com/symflower/eval-dev-quality/language" + "github.com/symflower/eval-dev-quality/log" ) func TestModelSymflowerGenerateTestsForFile(t *testing.T) { @@ -33,6 +33,13 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) { validate := func(t *testing.T, tc *testCase) { t.Run(tc.Name, func(t *testing.T) { + log, logger := log.Buffer() + defer func() { + if t.Failed() { + assert.Empty(t, log.String()) + } + }() + temporaryPath := t.TempDir() repositoryPath := filepath.Join(temporaryPath, filepath.Base(tc.RepositoryPath)) require.NoError(t, osutil.CopyTree(tc.RepositoryPath, repositoryPath)) @@ -44,7 +51,7 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) { if tc.ModelSymflower == nil { tc.ModelSymflower = &ModelSymflower{} } - actualAssessment, actualError := tc.ModelSymflower.GenerateTestsForFile(tc.Language, repositoryPath, tc.FilePath) + actualAssessment, actualError := tc.ModelSymflower.GenerateTestsForFile(logger, tc.Language, repositoryPath, tc.FilePath) if tc.ExpectedError != nil { assert.ErrorIs(t, tc.ExpectedError, actualError) @@ -53,7 +60,7 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) { } metricstesting.AssertAssessmentsEqual(t, tc.ExpectedAssessment, actualAssessment) - actualCoverage, err := tc.Language.Execute(repositoryPath) + actualCoverage, err := tc.Language.Execute(logger, repositoryPath) require.NoError(t, err) assert.Equal(t, tc.ExpectedCoverage, actualCoverage) }) diff --git a/provider/provider.go b/provider/provider.go index 10da33454..211ea752b 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -4,6 +4,7 @@ import ( "context" pkgerrors "github.com/pkg/errors" + "github.com/symflower/eval-dev-quality/model" ) diff --git a/util/exec.go b/util/exec.go index f99dc4c28..929516452 100644 --- a/util/exec.go +++ b/util/exec.go @@ -21,7 +21,7 @@ type Command struct { } // CommandWithResult executes a command, and prints and returns STDERR/STDOUT. -func CommandWithResult(command *Command) (stdout string, stderr string, err error) { +func CommandWithResult(log *log.Logger, command *Command) (stdout string, stderr string, err error) { log.Printf("$ %s", strings.Join(command.Command, " ")) var stdoutWriter bytes.Buffer