Skip to content

Commit

Permalink
Merge pull request #7 from symflower/rewrite-go-flags
Browse files Browse the repository at this point in the history
Rewrite binary with go-flags because it is much easier to extend
  • Loading branch information
bauersimon authored Apr 2, 2024
2 parents 8329b60 + ab2353e commit 1e9fdf4
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 63 deletions.
36 changes: 36 additions & 0 deletions cmd/eval-symflower-codegen-testing/cmd/command.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package cmd

import (
"log"
"os"

"github.com/jessevdk/go-flags"
)

// Command holds the root command.
type Command struct {
Evaluate `command:"evaluate" description:"Run an evaluation, by default with all defined models, repositories and tasks."`
}

// Execute executes the root command.
func Execute() {
var parser = flags.NewNamedParser("eval-symflower-codegen-testing", flags.Default)
parser.LongDescription = "Command to manage, update and actually execute the `eval-symflower-codegen-testing` evaluation benchmark."
if _, err := parser.AddGroup("Common command options", "", &Command{}); err != nil {
log.Fatalf("Could not add arguments group: %+v", err)
}

// Print the help, when there is no active command.
parser.SubcommandsOptional = true

if _, err := parser.Parse(); err != nil {
if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
return
}

log.Fatalf("Could not parse arguments: %+v", err)
}
if parser.Active == nil {
parser.WriteHelp(os.Stdout)
}
}
89 changes: 62 additions & 27 deletions cmd/eval-symflower-codegen-testing/cmd/evaluate.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,42 +4,77 @@ import (
"log"
"path/filepath"
"sort"
"strings"

"github.com/spf13/cobra"
"golang.org/x/exp/maps"

"github.com/symflower/eval-symflower-codegen-testing/evaluate"
"github.com/symflower/eval-symflower-codegen-testing/language"
"github.com/symflower/eval-symflower-codegen-testing/model"
"github.com/zimmski/osutil"
)

var commandEvalute = &cobra.Command{
Use: "evaluate",
Short: "Run an evaluation, by default with all defined models, repositories and tasks.",
Run: func(command *cobra.Command, arguments []string) {
// Gather languages.
languageIDs := maps.Keys(language.Languages)
sort.Strings(languageIDs)

// Gather models.
modelIDs := maps.Keys(model.Models)
sort.Strings(modelIDs)

// Check that models and languages can be evaluated by executing the "plain" repositories.
log.Printf("Checking that models and languages can used for evaluation")
for _, languageID := range languageIDs {
for _, modelID := range modelIDs {
model := model.Models[modelID]
language := language.Languages[languageID]

if err := evaluate.EvaluateRepository(model, language, filepath.Join("testdata", language.ID(), "plain")); err != nil {
log.Fatalf("%+v", err)
}
// Evaluate holds the "evaluation" command.
type Evaluate struct {
// Languages determines which language should be used for the evaluation, or empty if all languages should be used.
Languages []string `long:"language" description:"Evaluate with this language. By default all languages are used."`
// Models determines which models should be used for the evaluation, or empty if all models should be used.
Models []string `long:"model" description:"Evaluate with this model. By default all models are used."`
// TestdataPath determines the testdata path where all repositories reside grouped by languages.
TestdataPath string `long:"testdata" description:"Path to the testdata directory where all repositories reside grouped by languages." default:"testdata/"`
}

func (command *Evaluate) Execute(args []string) (err error) {
// Gather languages.
if len(command.Languages) == 0 {
command.Languages = maps.Keys(language.Languages)
} else {
for _, languageID := range command.Languages {
if _, ok := language.Languages[languageID]; !ok {
ls := maps.Keys(language.Languages)
sort.Strings(ls)

log.Fatalf("ERROR: language %s does not exist. Valid languages are: %s", languageID, strings.Join(ls, ", "))
}
}
},
}
}
sort.Strings(command.Languages)

// Gather models.
if len(command.Models) == 0 {
command.Models = maps.Keys(model.Models)
} else {
for _, modelID := range command.Models {
if _, ok := model.Models[modelID]; !ok {
ms := maps.Keys(model.Models)
sort.Strings(ms)

log.Fatalf("ERROR: model %s does not exist. Valid models are: %s", modelID, strings.Join(ms, ", "))
}
}
}
sort.Strings(command.Models)

if err := osutil.DirExists(command.TestdataPath); err != nil {
log.Fatalf("ERROR: testdata path %q cannot be accessed: %s", command.TestdataPath, err)
}
command.TestdataPath, err = filepath.Abs(command.TestdataPath)
if err != nil {
log.Fatalf("ERROR: could not resolve testdata path %q to an absolute path: %s", command.TestdataPath, err)
}

// Check that models and languages can be evaluated by executing the "plain" repositories.
log.Printf("Checking that models and languages can used for evaluation")
for _, languageID := range command.Languages {
for _, modelID := range command.Models {
model := model.Models[modelID]
language := language.Languages[languageID]

if err := evaluate.EvaluateRepository(model, language, filepath.Join(command.TestdataPath, language.ID(), "plain")); err != nil {
log.Fatalf("%+v", err)
}
}
}

func init() {
commandRoot.AddCommand(commandEvalute)
return nil
}
25 changes: 0 additions & 25 deletions cmd/eval-symflower-codegen-testing/cmd/root.go

This file was deleted.

4 changes: 1 addition & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module github.com/symflower/eval-symflower-codegen-testing
go 1.21.5

require (
github.com/jessevdk/go-flags v1.5.1-0.20210607101731-3927b71304df
github.com/pkg/errors v0.9.1
github.com/spf13/cobra v1.8.0
github.com/stretchr/testify v1.9.0
github.com/zimmski/osutil v1.1.0
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8
Expand All @@ -13,12 +13,10 @@ require (
require (
github.com/avast/retry-go v3.0.0+incompatible // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/schollz/progressbar/v3 v3.14.2 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae // indirect
github.com/ulikunitz/xz v0.5.11 // indirect
github.com/yuin/goldmark v1.7.0 // indirect
Expand Down
11 changes: 3 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jessevdk/go-flags v1.5.1-0.20210607101731-3927b71304df h1:JTDw/M13b6dZmEJI/vfcCLENqcjUHi9UBry+R0pjh5Q=
github.com/jessevdk/go-flags v1.5.1-0.20210607101731-3927b71304df/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
Expand All @@ -16,13 +15,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/schollz/progressbar/v3 v3.14.2 h1:EducH6uNLIWsr560zSV1KrTeUb/wZGAHqyMFIEa99ks=
github.com/schollz/progressbar/v3 v3.14.2/go.mod h1:aQAZQnhF4JGFtRJiw/eobaXpsqpVQAftEQ+hLGXaRc4=
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
Expand All @@ -37,6 +31,7 @@ github.com/zimmski/osutil v1.1.0 h1:laxALUxv1fHD+g/Db5swvl4FNYkh1SSGxtdqBiZziSA=
github.com/zimmski/osutil v1.1.0/go.mod h1:TZrA1ZvRIeylQ0ECaANmCVlT0WR/62zJxMdQX9SyLvY=
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw=
golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ=
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
Expand Down

0 comments on commit 1e9fdf4

Please sign in to comment.