Merge pull request #19 from symflower/rename-repository

Rename "eval-codegen-testing" to "eval-dev-quality" because the benchmark should be generic for all quality of development

Makefile

@@ -1,6 +1,6 @@
 export ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 
-export PACKAGE_BASE := github.com/symflower/eval-codegen-testing
+export PACKAGE_BASE := github.com/symflower/eval-dev-quality
 export UNIT_TEST_TIMEOUT := 480
 
 ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))

README.md

@@ -1 +1 @@
-# eval-codegen-testing
+# eval-dev-quality

cmd/eval-codegen-testing/cmd/command.go → cmd/eval-dev-quality/cmd/command.go

@@ -14,8 +14,8 @@ type Command struct {
 
 // Execute executes the root command.
 func Execute() {
-	var parser = flags.NewNamedParser("eval-codegen-testing", flags.Default)
-	parser.LongDescription = "Command to manage, update and actually execute the `eval-codegen-testing` evaluation benchmark."
+	var parser = flags.NewNamedParser("eval-dev-quality", flags.Default)
+	parser.LongDescription = "Command to manage, update and actually execute the `eval-dev-quality` evaluation benchmark."
 	if _, err := parser.AddGroup("Common command options", "", &Command{}); err != nil {
 		log.Fatalf("Could not add arguments group: %+v", err)
 	}

cmd/eval-codegen-testing/cmd/evaluate.go → cmd/eval-dev-quality/cmd/evaluate.go

@@ -10,12 +10,12 @@ import (
 	"github.com/zimmski/osutil"
 	"golang.org/x/exp/maps"
 
-	"github.com/symflower/eval-codegen-testing/evaluate"
-	"github.com/symflower/eval-codegen-testing/language"
-	"github.com/symflower/eval-codegen-testing/model"
-	"github.com/symflower/eval-codegen-testing/provider"
-	_ "github.com/symflower/eval-codegen-testing/provider/openrouter"
-	_ "github.com/symflower/eval-codegen-testing/provider/symflower"
+	"github.com/symflower/eval-dev-quality/evaluate"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/model"
+	"github.com/symflower/eval-dev-quality/provider"
+	_ "github.com/symflower/eval-dev-quality/provider/openrouter"
+	_ "github.com/symflower/eval-dev-quality/provider/symflower"
 )
 
 // Evaluate holds the "evaluation" command.

cmd/eval-dev-quality/main.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"github.com/symflower/eval-dev-quality/cmd/eval-dev-quality/cmd"
+)
+
+func main() {
+	cmd.Execute()
+}

evaluate/repository.go

@@ -9,8 +9,8 @@ import (
 	pkgerrors "github.com/pkg/errors"
 	"github.com/zimmski/osutil"
 
-	"github.com/symflower/eval-codegen-testing/language"
-	"github.com/symflower/eval-codegen-testing/model"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/model"
 )
 
 // EvaluateRepository evaluate a repository with the given model and language.
@@ -20,7 +20,7 @@ func EvaluateRepository(model model.Model, language language.Language, repositor
 		log.Printf("Evaluated model %q using language %q and repository %q: encountered %d problems", model.ID(), language.ID(), repositoryPath, len(problems))
 	}()
 
-	temporaryPath, err := os.MkdirTemp("", "eval-codegen-testing")
+	temporaryPath, err := os.MkdirTemp("", "eval-dev-quality")
 	if err != nil {
 		return metrics, problems, pkgerrors.WithStack(err)
 	}

go.mod

@@ -1,4 +1,4 @@
-module github.com/symflower/eval-codegen-testing
+module github.com/symflower/eval-dev-quality
 
 go 1.21.5
 

language/golang.go

@@ -12,7 +12,7 @@ import (
 	pkgerrors "github.com/pkg/errors"
 	"github.com/zimmski/osutil"
 
-	"github.com/symflower/eval-codegen-testing/util"
+	"github.com/symflower/eval-dev-quality/util"
 )
 
 // LanguageGolang holds a Go language to evaluate a repository.

model/llm/llm.go

@@ -11,10 +11,10 @@ import (
 	pkgerrors "github.com/pkg/errors"
 	"github.com/zimmski/osutil/bytesutil"
 
-	"github.com/symflower/eval-codegen-testing/language"
-	"github.com/symflower/eval-codegen-testing/model"
-	"github.com/symflower/eval-codegen-testing/model/llm/prompt"
-	"github.com/symflower/eval-codegen-testing/provider"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/model"
+	"github.com/symflower/eval-dev-quality/model/llm/prompt"
+	"github.com/symflower/eval-dev-quality/provider"
 )
 
 // llm represents a LLM model accessed via a provider.

model/llm/llm_test.go

@@ -11,8 +11,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/zimmski/osutil/bytesutil"
 
-	"github.com/symflower/eval-codegen-testing/language"
-	providertesting "github.com/symflower/eval-codegen-testing/provider/testing"
+	"github.com/symflower/eval-dev-quality/language"
+	providertesting "github.com/symflower/eval-dev-quality/provider/testing"
 )
 
 func TestModelLLMGenerateTestsForFile(t *testing.T) {

model/model.go

@@ -1,6 +1,6 @@
 package model
 
-import "github.com/symflower/eval-codegen-testing/language"
+import "github.com/symflower/eval-dev-quality/language"
 
 // Model defines a model that can be queried for generations.
 type Model interface {

model/symflower/symflower.go

@@ -3,10 +3,10 @@ package symflower
 import (
 	pkgerrors "github.com/pkg/errors"
 
-	"github.com/symflower/eval-codegen-testing/language"
-	"github.com/symflower/eval-codegen-testing/model"
-	"github.com/symflower/eval-codegen-testing/provider"
-	"github.com/symflower/eval-codegen-testing/util"
+	"github.com/symflower/eval-dev-quality/language"
+	"github.com/symflower/eval-dev-quality/model"
+	"github.com/symflower/eval-dev-quality/provider"
+	"github.com/symflower/eval-dev-quality/util"
 )
 
 // ModelSymflower holds a Symflower model using the locally installed CLI.

model/symflower/symflower_test.go

@@ -8,7 +8,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/zimmski/osutil"
 
-	"github.com/symflower/eval-codegen-testing/language"
+	"github.com/symflower/eval-dev-quality/language"
 )
 
 func TestModelSymflowerGenerateTestsForFile(t *testing.T) {

provider/openrouter/openrouter.go

@@ -8,9 +8,9 @@ import (
 	pkgerrors "github.com/pkg/errors"
 	"github.com/sashabaranov/go-openai"
 
-	"github.com/symflower/eval-codegen-testing/model"
-	"github.com/symflower/eval-codegen-testing/model/llm"
-	"github.com/symflower/eval-codegen-testing/provider"
+	"github.com/symflower/eval-dev-quality/model"
+	"github.com/symflower/eval-dev-quality/model/llm"
+	"github.com/symflower/eval-dev-quality/provider"
 )
 
 type openRouterProvider struct {

provider/provider.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	pkgerrors "github.com/pkg/errors"
-	"github.com/symflower/eval-codegen-testing/model"
+	"github.com/symflower/eval-dev-quality/model"
 )
 
 // ProviderModelSeparator is the the separator between a provider and a model.

provider/symflower/symflower.go

@@ -1,9 +1,9 @@
 package symflower
 
 import (
-	"github.com/symflower/eval-codegen-testing/model"
-	"github.com/symflower/eval-codegen-testing/model/symflower"
-	"github.com/symflower/eval-codegen-testing/provider"
+	"github.com/symflower/eval-dev-quality/model"
+	"github.com/symflower/eval-dev-quality/model/symflower"
+	"github.com/symflower/eval-dev-quality/provider"
 )
 
 type symflowerProvider struct{}

provider/testing/mock.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/stretchr/testify/mock"
 
-	provider "github.com/symflower/eval-codegen-testing/provider"
+	provider "github.com/symflower/eval-dev-quality/provider"
 )
 
 // MockQueryProvider is a mocked QueryProvider.