Merge pull request #13 from symflower/dynamic-langugage-name-llm-prompt

Dynamic language name in LLM prompt

evaluate/repository.go

@@ -45,7 +45,7 @@ func EvaluateRepository(model model.Model, language language.Language, repositor
 
 	for _, filePath := range filePaths {
 		metrics.Total++
-		if err := model.GenerateTestsForFile(temporaryRepositoryPath, filePath); err != nil {
+		if err := model.GenerateTestsForFile(language, temporaryRepositoryPath, filePath); err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, filePath))
 			metrics.Problems++
 

language/golang.go

@@ -29,6 +29,11 @@ func (language *LanguageGolang) ID() (id string) {
 	return "golang"
 }
 
+// Name is the prose name of this language.
+func (language *LanguageGolang) Name() (id string) {
+	return "Go"
+}
+
 // Files returns a list of relative file paths of the repository that should be evaluated.
 func (language *LanguageGolang) Files(repositoryPath string) (filePaths []string, err error) {
 	repositoryPath, err = filepath.Abs(repositoryPath)

language/language.go

@@ -8,6 +8,8 @@ import (
 type Language interface {
 	// ID returns the unique ID of this language.
 	ID() (id string)
+	// Name is the prose name of this language.
+	Name() (id string)
 
 	// Files returns a list of relative file paths of the repository that should be evaluated.
 	Files(repositoryPath string) (filePaths []string, err error)

model/llm/llm.go

@@ -11,6 +11,7 @@ import (
 	pkgerrors "github.com/pkg/errors"
 	"github.com/zimmski/osutil/bytesutil"
 
+	"github.com/symflower/eval-symflower-codegen-testing/language"
 	"github.com/symflower/eval-symflower-codegen-testing/model"
 	"github.com/symflower/eval-symflower-codegen-testing/model/llm/prompt"
 	"github.com/symflower/eval-symflower-codegen-testing/provider"
@@ -34,6 +35,9 @@ func NewLLMModel(provider provider.QueryProvider, modelIdentifier string) model.
 
 // llmGenerateTestForFilePromptContext is the context for template for generating an LLM test generation prompt.
 type llmGenerateTestForFilePromptContext struct {
+	// Language holds the programming language name.
+	Language language.Language
+
 	// Code holds the source code of the file.
 	Code string
 	// FilePath holds the file path of the file.
@@ -44,11 +48,11 @@ type llmGenerateTestForFilePromptContext struct {
 
 // llmGenerateTestForFilePromptTemplate is the template for generating an LLM test generation prompt.
 var llmGenerateTestForFilePromptTemplate = template.Must(template.New("model-llm-generate-test-for-file-prompt").Parse(bytesutil.StringTrimIndentations(`
-	Given the following Go code file "{{ .FilePath }}" with package "{{ .ImportPath }}", provide a test file for this code.
+	Given the following {{ .Language.Name }} code file "{{ .FilePath }}" with package "{{ .ImportPath }}", provide a test file for this code.
 	The tests should produce 100 percent code coverage and must compile.
 	The response must contain only the test code and nothing else.
 
-	` + "```" + `
+	` + "```" + `{{ .Language.ID }}
 	{{ .Code }}
 	` + "```" + `
 `)))
@@ -73,7 +77,7 @@ func (m *llm) ID() (id string) {
 }
 
 // GenerateTestsForFile generates test files for the given implementation file in a repository.
-func (m *llm) GenerateTestsForFile(repositoryPath string, filePath string) (err error) {
+func (m *llm) GenerateTestsForFile(language language.Language, repositoryPath string, filePath string) (err error) {
 	data, err := os.ReadFile(filepath.Join(repositoryPath, filePath))
 	if err != nil {
 		return err
@@ -83,6 +87,8 @@ func (m *llm) GenerateTestsForFile(repositoryPath string, filePath string) (err
 	importPath := filepath.Join(filepath.Base(repositoryPath), filepath.Dir(filePath))
 
 	message, err := llmGenerateTestForFilePrompt(&llmGenerateTestForFilePromptContext{
+		Language: language,
+
 		Code:       fileContent,
 		FilePath:   filePath,
 		ImportPath: importPath,

model/llm/llm_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/zimmski/osutil/bytesutil"
 
+	"github.com/symflower/eval-symflower-codegen-testing/language"
 	providertesting "github.com/symflower/eval-symflower-codegen-testing/provider/testing"
 )
 
@@ -20,9 +21,10 @@ func TestModelLLMGenerateTestsForFile(t *testing.T) {
 
 		SetupMock func(mockedProvider *providertesting.MockQueryProvider)
 
+		Language          language.Language
+		ModelID           string
 		SourceFileContent string
 		SourceFilePath    string
-		ModelID           string
 
 		ExpectedTestFileContent string
 		ExpectedTestFilePath    string
@@ -40,7 +42,7 @@ func TestModelLLMGenerateTestsForFile(t *testing.T) {
 			tc.SetupMock(mock)
 			llm := NewLLMModel(mock, tc.ModelID)
 
-			assert.NoError(t, llm.GenerateTestsForFile(temporaryPath, tc.SourceFilePath))
+			assert.NoError(t, llm.GenerateTestsForFile(tc.Language, temporaryPath, tc.SourceFilePath))
 
 			actualTestFileContent, err := os.ReadFile(filepath.Join(temporaryPath, tc.ExpectedTestFilePath))
 			assert.NoError(t, err)
@@ -56,6 +58,8 @@ func TestModelLLMGenerateTestsForFile(t *testing.T) {
 	`
 	sourceFilePath := "simple.go"
 	promptMessage, err := llmGenerateTestForFilePrompt(&llmGenerateTestForFilePromptContext{
+		Language: &language.LanguageGolang{},
+
 		Code:       bytesutil.StringTrimIndentations(sourceFileContent),
 		FilePath:   sourceFilePath,
 		ImportPath: "native",
@@ -74,9 +78,10 @@ func TestModelLLMGenerateTestsForFile(t *testing.T) {
 				`), nil)
 		},
 
+		Language:          &language.LanguageGolang{},
+		ModelID:           "model-id",
 		SourceFileContent: sourceFileContent,
 		SourceFilePath:    sourceFilePath,
-		ModelID:           "model-id",
 
 		ExpectedTestFileContent: `
 			package native

model/model.go

@@ -1,10 +1,12 @@
 package model
 
+import "github.com/symflower/eval-symflower-codegen-testing/language"
+
 // Model defines a model that can be queried for generations.
 type Model interface {
 	// ID returns the unique ID of this model.
 	ID() (id string)
 
 	// GenerateTestsForFile generates test files for the given implementation file in a repository.
-	GenerateTestsForFile(repositoryPath string, filePath string) (err error)
+	GenerateTestsForFile(language language.Language, repositoryPath string, filePath string) (err error)
 }

model/symflower/symflower.go

@@ -3,6 +3,7 @@ package symflower
 import (
 	pkgerrors "github.com/pkg/errors"
 
+	"github.com/symflower/eval-symflower-codegen-testing/language"
 	"github.com/symflower/eval-symflower-codegen-testing/model"
 	"github.com/symflower/eval-symflower-codegen-testing/provider"
 	"github.com/symflower/eval-symflower-codegen-testing/util"
@@ -19,7 +20,7 @@ func (m *ModelSymflower) ID() (id string) {
 }
 
 // GenerateTestsForFile generates test files for the given implementation file in a repository.
-func (m *ModelSymflower) GenerateTestsForFile(repositoryPath string, filePath string) (err error) {
+func (m *ModelSymflower) GenerateTestsForFile(language language.Language, repositoryPath string, filePath string) (err error) {
 	_, _, err = util.CommandWithResult(&util.Command{
 		Command: []string{
 			"symflower", "unit-tests",

model/symflower/symflower_test.go

@@ -40,7 +40,7 @@ func TestModelSymflowerGenerateTestsForFile(t *testing.T) {
 			if tc.ModelSymflower == nil {
 				tc.ModelSymflower = &ModelSymflower{}
 			}
-			actualErr := tc.ModelSymflower.GenerateTestsForFile(repositoryPath, tc.FilePath)
+			actualErr := tc.ModelSymflower.GenerateTestsForFile(tc.Language, repositoryPath, tc.FilePath)
 
 			if tc.ExpectedError != nil {
 				assert.ErrorIs(t, tc.ExpectedError, actualErr)