diff --git a/README.md b/README.md index 4a91042..a8e6bde 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,12 @@ And done. Based on the recipe, the output will be printed to the std out, or in ```zsh ... -{ - "email": "valor@github.com", - "is_active": true, - "membership": "premium" -} +--------------------------- +example/resource/valor.json +--------------------------- +email: valor@github.com +is_active: true +membership: premium ... ``` @@ -76,8 +77,8 @@ What Valor does is actually stated in the recipe file `valor.yaml`. Behind the s 2. execute framework pointed by field `framework_names` 3. in the framework, run validation based on `schemas` 4. if no error is found, load the required definition under `definitions` -5. if no error is found, execute proceduresstated under `procedures` -6. if no error is found, write the result based on `output_targets` +5. if no error is found, execute procedures stated under `procedures` +6. if no error is found, write the result based on `output` The explanation here is quite brief. For further explanation, try checkout the documentation [here](#documentation). diff --git a/cmd/execute.go b/cmd/execute.go index 99dad44..e9bb985 100644 --- a/cmd/execute.go +++ b/cmd/execute.go @@ -17,7 +17,7 @@ import ( const ( defaultBatchSize = 4 - defaultProgressType = "verbose" + defaultProgressType = "progressive" ) var ( @@ -30,10 +30,7 @@ func getExecuteCmd() *cobra.Command { Use: "execute", Short: "Execute pipeline based on the specified recipe", RunE: func(cmd *cobra.Command, args []string) error { - if err := executePipeline(recipePath, batchSize, progressType, nil); err != nil { - return errors.New(string(err.JSON())) - } - return nil + return executePipeline(recipePath, batchSize, progressType, nil) }, } runCmd.PersistentFlags().StringVarP(&recipePath, "recipe-path", "R", defaultRecipePath, "Path of the recipe file") @@ -44,7 +41,7 @@ func getExecuteCmd() *cobra.Command { return runCmd } -func executePipeline(recipePath string, batchSize int, progressType string, enrich func(*recipe.Recipe) model.Error) model.Error { +func executePipeline(recipePath string, batchSize int, progressType string, enrich func(*recipe.Recipe) error) error { rcp, err := loadRecipe(recipePath, defaultRecipeType, defaultRecipeFormat) if err != nil { return err @@ -62,14 +59,15 @@ func executePipeline(recipePath string, batchSize int, progressType string, enri return err } evaluate := getEvaluate() - pipeline, err := core.NewPipeline(rcp, batchSize, evaluate, newProgress) + pipeline, err := core.NewPipeline(rcp, evaluate, batchSize, newProgress) if err != nil { return err } - if err := pipeline.Execute(); err != nil { - return err + err = pipeline.Execute() + if e, ok := err.(*model.Error); ok { + return errors.New(string(e.JSON())) } - return nil + return err } func getEvaluate() model.Evaluate { @@ -79,7 +77,7 @@ func getEvaluate() model.Evaluate { } } -func loadRecipe(path, _type, format string) (*recipe.Recipe, model.Error) { +func loadRecipe(path, _type, format string) (*recipe.Recipe, error) { fnReader, err := io.Readers.Get(_type) if err != nil { return nil, err @@ -87,24 +85,17 @@ func loadRecipe(path, _type, format string) (*recipe.Recipe, model.Error) { getPath := func() string { return path } - filterPath := func(p string) bool { - return true - } - postProcess := func(p string, c []byte) (*model.Data, model.Error) { + postProcess := func(p string, c []byte) (*model.Data, error) { return &model.Data{ Content: bytes.ToLower(c), Path: p, Type: format, }, nil } - reader := fnReader(getPath, filterPath, postProcess) + reader := fnReader(getPath, postProcess) decode, err := endec.Decodes.Get(format) if err != nil { return nil, err } - rcp, err := recipe.Load(reader, decode) - if err != nil { - return nil, err - } - return rcp, nil + return recipe.Load(reader, decode) } diff --git a/cmd/profie.go b/cmd/profile.go similarity index 91% rename from cmd/profie.go rename to cmd/profile.go index b515284..513e590 100644 --- a/cmd/profie.go +++ b/cmd/profile.go @@ -1,7 +1,6 @@ package cmd import ( - "errors" "fmt" "os" @@ -18,7 +17,7 @@ func getProfileCmd() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { rcp, err := loadRecipe(recipePath, defaultRecipeType, defaultRecipeFormat) if err != nil { - return errors.New(string(err.JSON())) + return err } fmt.Println("RESOURCE:") @@ -66,8 +65,12 @@ func getFrameworkTable(rcp *recipe.Recipe) *tablewriter.Table { for _, p := range f.Procedures { table.Append([]string{f.Name, "procedure", p.Name}) } - for _, o := range f.OutputTargets { - table.Append([]string{f.Name, "output", o.Name}) + for _, p := range f.Procedures { + if p.Output != nil { + for _, t := range p.Output.Targets { + table.Append([]string{f.Name, "output", t.Path}) + } + } } } return table diff --git a/cmd/resource.go b/cmd/resource.go index 9ff9aeb..77ad369 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -28,7 +28,7 @@ func getResourceCmd() *cobra.Command { Use: "resource", Short: "Execute pipeline for a specific resource", RunE: func(cmd *cobra.Command, args []string) error { - enrich := func(rcp *recipe.Recipe) model.Error { + enrich := func(rcp *recipe.Recipe) error { return enrichRecipe(rcp, &resourceArg{ Name: name, Format: format, @@ -36,10 +36,11 @@ func getResourceCmd() *cobra.Command { Path: path, }) } - if err := executePipeline(recipePath, batchSize, progressType, enrich); err != nil { - return errors.New(string(err.JSON())) + err := executePipeline(recipePath, batchSize, progressType, enrich) + if e, ok := err.(*model.Error); ok { + return errors.New(string(e.JSON())) } - return nil + return err }, } resourceCmd.Flags().StringVarP(&name, "name", "n", "", "name of the resource recipe to be used") @@ -51,7 +52,7 @@ func getResourceCmd() *cobra.Command { return resourceCmd } -func enrichRecipe(rcp *recipe.Recipe, arg *resourceArg) model.Error { +func enrichRecipe(rcp *recipe.Recipe, arg *resourceArg) error { if arg.Name == "" { return nil } @@ -71,12 +72,8 @@ func enrichRecipe(rcp *recipe.Recipe, arg *resourceArg) model.Error { break } } - const defaultErrKey = "enrichRecipe" if resourceRcp == nil { - return model.BuildError( - defaultErrKey, - fmt.Errorf("resource recipe [%s] is not found", arg.Name), - ) + return fmt.Errorf("resource recipe [%s] is not found", arg.Name) } rcp.Resources = []*recipe.Resource{resourceRcp} return nil diff --git a/core/core.go b/core/core.go index 2b18993..e262e99 100644 --- a/core/core.go +++ b/core/core.go @@ -3,30 +3,25 @@ package core import ( "errors" "fmt" - "path" "strings" "sync" "github.com/gojek/optimus-extension-valor/model" _ "github.com/gojek/optimus-extension-valor/plugin/io" // init error writer "github.com/gojek/optimus-extension-valor/recipe" - "github.com/gojek/optimus-extension-valor/registry/formatter" "github.com/gojek/optimus-extension-valor/registry/io" ) -const ( - errorWriterType = "std" - defaultProgressType = "verbose" -) +const errorWriterType = "std" var errorWriter model.Writer func init() { - writer, err := io.Writers.Get(errorWriterType) + writerFn, err := io.Writers.Get(errorWriterType) if err != nil { panic(err) } - errorWriter = writer + errorWriter = writerFn(model.TreatmentError) } const ( @@ -53,13 +48,21 @@ type Pipeline struct { // NewPipeline initializes pipeline process func NewPipeline( rcp *recipe.Recipe, - batchSize int, evaluate model.Evaluate, + batchSize int, newProgress model.NewProgress, -) (*Pipeline, model.Error) { - const defaultErrKey = "NewPipeline" +) (*Pipeline, error) { if rcp == nil { - return nil, model.BuildError(defaultErrKey, errors.New("recipe is nil")) + return nil, errors.New("recipe is nil") + } + if evaluate == nil { + return nil, errors.New("evaluate function is nil") + } + if batchSize < 0 { + return nil, errors.New("batch size should be at least zero") + } + if newProgress == nil { + return nil, errors.New("new progress function is nil") } nameToFrameworkRecipe := make(map[string]*recipe.Framework) for _, frameworkRcp := range rcp.Frameworks { @@ -76,284 +79,215 @@ func NewPipeline( } // Execute executes pipeline process -func (p *Pipeline) Execute() model.Error { - const defaultErrKey = "Execute" +func (p *Pipeline) Execute() error { for _, resourceRcp := range p.recipe.Resources { - decorate := strings.Repeat("=", 12) - fmt.Printf("%s PROCESSING RESOURCE [%s] %s\n", decorate, resourceRcp.Name, decorate) - - fmt.Println("> Loading resource") - resource, err := p.loader.LoadResource(resourceRcp) + fmt.Printf("Resource [%s]\n", strings.ToUpper(resourceRcp.Name)) + fmt.Println("o> validating framework names") + if err := p.validateFrameworkNames(resourceRcp); err != nil { + return err + } + fmt.Println("o> loading the required framework data") + nameToFramework, err := p.getFrameworkNameToFramework(resourceRcp) if err != nil { - fmt.Println("* Loading failed!!!") return err } - fmt.Printf("> Loading finished\n") - - for _, frameworkName := range resource.FrameworkNames { - decorate := strings.Repeat(":", 5) - fmt.Printf("%s Processing Framework [%s] %s\n", decorate, frameworkName, decorate) - frameworkRcp := p.nameToFrameworkRecipe[frameworkName] - - fmt.Println(" >> Loading framework") - framework, err := p.loader.LoadFramework(frameworkRcp) - if err != nil { - fmt.Println(" ** Loading failed!!!") - return err - } - fmt.Printf(" > Loading finished\n") - - fmt.Println(" >> Validating resource") - success := p.executeValidate(framework, resource.ListOfData) - if !success { - fmt.Println(" ** Validation failed!!!") - key := fmt.Sprintf("%s [validate: %s]", defaultErrKey, frameworkName) - return model.BuildError(key, errors.New("error is met during validation")) - } - fmt.Printf(" > Validation finished\n") - - fmt.Println(" >> Evaluating resource") - evalResults, success := p.executeEvaluate(framework, resource.ListOfData) - if !success { - fmt.Println(" ** Evaluation failed!!!") - key := fmt.Sprintf("%s [evaluate: %s]", defaultErrKey, frameworkName) - return model.BuildError(key, errors.New("error is met during evaluation")) - } - fmt.Printf(" > Evaluation finished\n") - - fmt.Println(" >> Writing result") - success = p.writeOutput(evalResults, framework.OutputTargets) - if !success { - fmt.Println(" ** Writing failed!!!") - key := fmt.Sprintf("%s [write: %s]", defaultErrKey, frameworkName) - return model.BuildError(key, errors.New("error is met during write")) - } - fmt.Printf(" > Writing finished\n") + fmt.Println("o> loading the required validator") + nameToValidator, err := p.getFrameworkNameToValidator(nameToFramework) + if err != nil { + return err + } + fmt.Println("o> loading the required evaluator") + nameToEvaluator, err := p.getFrameworkNameToEvaluator(nameToFramework) + if err != nil { + return err + } + fmt.Println("o> executing resource") + if err := p.executeResource(resourceRcp, nameToValidator, nameToEvaluator); err != nil { + return err } fmt.Println() } return nil } -func (p *Pipeline) executeValidate(framework *model.Framework, resourceData []*model.Data) bool { - const defaultErrKey = "validate" - validator, err := NewValidator(framework) - if err != nil { - errorWriter.Write(&model.Data{ - Type: errorWriterType, - Content: err.JSON(), - Path: defaultErrKey, - }) - return false +func (p *Pipeline) executeResource(resourceRcp *recipe.Resource, nameToValidator map[string]*Validator, nameToEvaluator map[string]*Evaluator) error { + if resourceRcp == nil { + return errors.New("resource recipe is nil") } - progress := p.newProgress(fmt.Sprintf("%s [%s]", defaultErrKey, framework.Name), len(resourceData)) - wg := &sync.WaitGroup{} - mtx := &sync.Mutex{} - - success := true - for i, data := range resourceData { - wg.Add(1) - go func(idx int, v *Validator, w *sync.WaitGroup, m *sync.Mutex, d *model.Data) { - defer w.Done() - if err := v.Validate(d); err != nil { - m.Lock() - success = false - m.Unlock() - - pt := fmt.Sprintf("%s [%d]", defaultErrKey, idx) - if d != nil { - pt = fmt.Sprintf("%s [%s]", defaultErrKey, d.Path) - } - errorWriter.Write(&model.Data{ - Type: errorWriterType, - Content: err.JSON(), - Path: pt, - }) - } - if success { - m.Lock() - progress.Increment() - m.Unlock() - } - }(i, validator, wg, mtx, data) - } - wg.Wait() - progress.Wait() - return success -} - -func (p *Pipeline) executeEvaluate(framework *model.Framework, resourceData []*model.Data) ([]*model.Data, bool) { - const defaultErrKey = "evaluate" - evaluator, err := NewEvaluator(framework, p.evaluate) + resourcePaths, err := ExplorePaths(resourceRcp.Path, resourceRcp.Type, resourceRcp.Format) if err != nil { - errorWriter.Write(&model.Data{ - Type: errorWriterType, - Content: err.JSON(), - Path: defaultErrKey, - }) - return nil, false + return err } - progress := p.newProgress(fmt.Sprintf("%s [%s]", defaultErrKey, framework.Name), len(resourceData)) + outputError := &model.Error{} + progress := p.newProgress(resourceRcp.Name, len(resourcePaths)) batch := p.batchSize - if batch <= 0 || batch >= len(resourceData) { - batch = len(resourceData) + if batch == 0 || batch >= len(resourcePaths) { + batch = len(resourcePaths) } counter := 0 + for counter < len(resourcePaths) { + wg := &sync.WaitGroup{} + mtx := &sync.Mutex{} + + for i := 0; i < batch && counter+i < len(resourcePaths); i++ { + wg.Add(1) - success := true - outputResult := make([]*model.Data, len(resourceData)) - for counter < len(resourceData) { - evalChans := make([]chan *model.Data, batch) - for i := 0; i < batch && counter+i < len(resourceData); i++ { - data := resourceData[counter+i] - ch := make(chan *model.Data) - go func(idx int, v *Evaluator, d *model.Data, c chan *model.Data) { - rst, err := v.Evaluate(d) - var output *model.Data + idx := counter + i + go func(pt string, w *sync.WaitGroup, m *sync.Mutex) { + defer w.Done() + data, err := p.loader.LoadData(pt, resourceRcp.Type, resourceRcp.Format) if err != nil { - pt := fmt.Sprintf("%s [%d]", defaultErrKey, idx) - if d != nil { - pt = d.Path + outputError.Add(pt, err) + return + } + for _, frameworkName := range resourceRcp.FrameworkNames { + handleErr := func(process, name string, success bool, err error) bool { + if err != nil { + var message string + if e, ok := err.(*model.Error); ok { + message = string(e.JSON()) + } else { + message = err.Error() + } + errorWriter.Write(&model.Data{ + Type: "std", + Path: pt, + Content: []byte(message), + }) + outputError.Add(pt, + fmt.Errorf("%s on framework [%s] encountered execution error", + process, name, + ), + ) + return false + } + if !success { + outputError.Add(pt, + fmt.Errorf("%s on framework [%s] encountered business error", + process, name, + ), + ) + return false + } + return true } - errorWriter.Write(&model.Data{ - Type: errorWriterType, - Content: err.JSON(), - Path: pt, - }) - } else { - output = &model.Data{ - Type: d.Type, - Path: d.Path, - Content: []byte(rst), + validator := nameToValidator[frameworkName] + if validator != nil { + success, err := validator.Validate(data) + if ok := handleErr("validation", frameworkName, success, err); !ok { + return + } + } + evaluator := nameToEvaluator[frameworkName] + if evaluator != nil { + success, err := evaluator.Evaluate(data) + if ok := handleErr("evaluation", frameworkName, success, err); !ok { + return + } } } - ch <- output - }(i, evaluator, data, ch) - evalChans[i] = ch + }(resourcePaths[idx], wg, mtx) } - for i := 0; i < batch && counter+i < len(resourceData); i++ { - rst := <-evalChans[i] - close(evalChans[i]) - if rst == nil { - success = false - } else { - outputResult[counter+i] = rst - } - progress.Increment() + wg.Wait() + + increment := batch + if increment+counter > len(resourcePaths) { + increment = len(resourcePaths) - counter } + progress.Increase(increment) counter += batch } progress.Wait() - - if success { - return outputResult, true + if outputError.Length() > 0 { + return outputError } - return nil, false + return nil } -func (p *Pipeline) writeOutput(evalResults []*model.Data, outputTargets []*model.OutputTarget) bool { - const defaultErrKey = "writeOutput" - formatters, err := p.getOutputFormatter(outputTargets) - if err != nil { - errorWriter.Write(&model.Data{ - Type: errorWriterType, - Content: err.JSON(), - Path: defaultErrKey, - }) - return false +func (p *Pipeline) getFrameworkNameToValidator(nameToFramework map[string]*model.Framework) (map[string]*Validator, error) { + outputValidator := make(map[string]*Validator) + outputError := &model.Error{} + for name, framework := range nameToFramework { + validator, err := NewValidator(framework) + if err != nil { + outputError.Add(name, err) + } else { + outputValidator[name] = validator + } } - writers, err := p.getOutputWriter(outputTargets) - if err != nil { - errorWriter.Write(&model.Data{ - Type: errorWriterType, - Content: err.JSON(), - Path: defaultErrKey, - }) - return false + if outputError.Length() > 0 { + return nil, outputError } + return outputValidator, nil +} + +func (p *Pipeline) getFrameworkNameToEvaluator(nameToFramework map[string]*model.Framework) (map[string]*Evaluator, error) { wg := &sync.WaitGroup{} mtx := &sync.Mutex{} - success := true - for i := 0; i < len(outputTargets); i++ { + outputEvaluator := make(map[string]*Evaluator) + outputError := &model.Error{} + for name, framework := range nameToFramework { wg.Add(1) - go func(idx int, w *sync.WaitGroup, m *sync.Mutex) { + + go func(n string, f *model.Framework, w *sync.WaitGroup, m *sync.Mutex) { defer w.Done() - newResults := make([]*model.Data, len(evalResults)) - currentSuccess := true - for j := 0; j < len(evalResults); j++ { - newContent, err := formatters[idx](evalResults[j].Content) - if err != nil { - errorWriter.Write(&model.Data{ - Type: evalResults[j].Path, - Path: evalResults[j].Path, - Content: err.JSON(), - }) - currentSuccess = false - continue - } - newResults[j] = &model.Data{ - Type: evalResults[j].Type, - Path: path.Join(outputTargets[idx].Path, evalResults[j].Path), - Content: newContent, - } - } - if currentSuccess { - if err := writers[idx].Write(newResults...); err != nil { - currentSuccess = false - errorWriter.Write(&model.Data{ - Type: errorWriterType, - Path: fmt.Sprintf("%s [%d]", defaultErrKey, idx), - Content: err.JSON(), - }) - } - } - if !currentSuccess { + evaluator, err := NewEvaluator(f, p.evaluate) + if err != nil { + outputError.Add(n, err) + } else { m.Lock() - success = false + outputEvaluator[n] = evaluator m.Unlock() } - }(i, wg, mtx) + }(name, framework, wg, mtx) + } + if outputError.Length() > 0 { + return nil, outputError } wg.Wait() - return success + return outputEvaluator, nil } -func (p *Pipeline) getOutputWriter(target []*model.OutputTarget) ([]model.Writer, model.Error) { - const defaultErrKey = "getOutputFormatter" - outputWriter := make([]model.Writer, len(target)) - outputError := make(model.Error) - for i, t := range target { - fn, err := io.Writers.Get(t.Type) - if err != nil { - key := fmt.Sprintf("%s [%d]", defaultErrKey, i) - outputError[key] = err - continue - } - outputWriter[i] = fn +func (p *Pipeline) getFrameworkNameToFramework(rcp *recipe.Resource) (map[string]*model.Framework, error) { + wg := &sync.WaitGroup{} + mtx := &sync.Mutex{} + + nameToFramework := make(map[string]*model.Framework) + outputError := &model.Error{} + for _, frameworkName := range rcp.FrameworkNames { + wg.Add(1) + + go func(frameworkRcp *recipe.Framework, w *sync.WaitGroup, m *sync.Mutex) { + defer w.Done() + framework, err := p.loader.LoadFramework(frameworkRcp) + if err != nil { + outputError.Add(frameworkRcp.Name, err) + } else { + m.Lock() + nameToFramework[frameworkRcp.Name] = framework + m.Unlock() + } + }(p.nameToFrameworkRecipe[frameworkName], wg, mtx) } - if len(outputError) > 0 { + wg.Wait() + + if outputError.Length() > 0 { return nil, outputError } - return outputWriter, nil + return nameToFramework, nil } -func (p *Pipeline) getOutputFormatter(target []*model.OutputTarget) ([]model.Format, model.Error) { - const defaultErrKey = "getOutputFormatter" - outputFormat := make([]model.Format, len(target)) - outputError := make(model.Error) - for i, t := range target { - fn, err := formatter.Formats.Get(jsonFormat, t.Format) - if err != nil { - key := fmt.Sprintf("%s [%d]", defaultErrKey, i) - outputError[key] = err - continue +func (p *Pipeline) validateFrameworkNames(resourceRcp *recipe.Resource) error { + outputError := &model.Error{} + for _, frameworkName := range resourceRcp.FrameworkNames { + if p.nameToFrameworkRecipe[frameworkName] == nil { + outputError.Add(frameworkName, fmt.Errorf("not found for resource [%s]", resourceRcp.Name)) } - outputFormat[i] = fn } - if len(outputError) > 0 { - return nil, outputError + if outputError.Length() > 0 { + return outputError } - return outputFormat, nil + return nil } diff --git a/core/core_test.go b/core/core_test.go new file mode 100644 index 0000000..1fc6524 --- /dev/null +++ b/core/core_test.go @@ -0,0 +1,89 @@ +package core_test + +import ( + "testing" + + "github.com/gojek/optimus-extension-valor/core" + "github.com/gojek/optimus-extension-valor/model" + "github.com/gojek/optimus-extension-valor/recipe" + + "github.com/stretchr/testify/assert" +) + +func TestNewPipeline(t *testing.T) { + t.Run("should return nil and error if recipe is nil", func(t *testing.T) { + var rcp *recipe.Recipe = nil + var evaluate model.Evaluate = func(name, snippet string) (string, error) { + return "", nil + } + var batchSize = 0 + var newProgress model.NewProgress = func(name string, total int) model.Progress { + return nil + } + + actualPipeline, actualErr := core.NewPipeline(rcp, evaluate, batchSize, newProgress) + + assert.Nil(t, actualPipeline) + assert.NotNil(t, actualErr) + }) + + t.Run("should return nil and error if evaluate is nil", func(t *testing.T) { + var rcp *recipe.Recipe = &recipe.Recipe{} + var evaluate model.Evaluate = nil + var batchSize = 0 + var newProgress model.NewProgress = func(name string, total int) model.Progress { + return nil + } + + actualPipeline, actualErr := core.NewPipeline(rcp, evaluate, batchSize, newProgress) + + assert.Nil(t, actualPipeline) + assert.NotNil(t, actualErr) + }) + + t.Run("should return nil and error if batchSize is less than zero", func(t *testing.T) { + var rcp *recipe.Recipe = &recipe.Recipe{} + var evaluate model.Evaluate = func(name, snippet string) (string, error) { + return "", nil + } + var batchSize = -1 + var newProgress model.NewProgress = func(name string, total int) model.Progress { + return nil + } + + actualPipeline, actualErr := core.NewPipeline(rcp, evaluate, batchSize, newProgress) + + assert.Nil(t, actualPipeline) + assert.NotNil(t, actualErr) + }) + + t.Run("should return nil and error if newProgress is nil", func(t *testing.T) { + var rcp *recipe.Recipe = &recipe.Recipe{} + var evaluate model.Evaluate = func(name, snippet string) (string, error) { + return "", nil + } + var batchSize = 0 + var newProgress model.NewProgress = nil + + actualPipeline, actualErr := core.NewPipeline(rcp, evaluate, batchSize, newProgress) + + assert.Nil(t, actualPipeline) + assert.NotNil(t, actualErr) + }) + + t.Run("should return pipeline and nil if no error is encountered", func(t *testing.T) { + var rcp *recipe.Recipe = &recipe.Recipe{} + var evaluate model.Evaluate = func(name, snippet string) (string, error) { + return "", nil + } + var batchSize = 0 + var newProgress model.NewProgress = func(name string, total int) model.Progress { + return nil + } + + actualPipeline, actualErr := core.NewPipeline(rcp, evaluate, batchSize, newProgress) + + assert.NotNil(t, actualPipeline) + assert.Nil(t, actualErr) + }) +} diff --git a/core/evaluator.go b/core/evaluator.go index 5880f37..cd627b2 100644 --- a/core/evaluator.go +++ b/core/evaluator.go @@ -7,7 +7,6 @@ import ( "sync" "github.com/gojek/optimus-extension-valor/model" - "github.com/gojek/optimus-extension-valor/registry/endec" ) // Evaluator contains information on how to evaluate a Resource @@ -18,13 +17,12 @@ type Evaluator struct { } // NewEvaluator initializes Evaluator -func NewEvaluator(framework *model.Framework, evaluate model.Evaluate) (*Evaluator, model.Error) { - const defaultErrKey = "NewEvaluator" +func NewEvaluator(framework *model.Framework, evaluate model.Evaluate) (*Evaluator, error) { if framework == nil { - return nil, model.BuildError(defaultErrKey, errors.New("framework is nil")) + return nil, errors.New("framework is nil") } if evaluate == nil { - return nil, model.BuildError(defaultErrKey, errors.New("evaluate is nil")) + return nil, errors.New("evaluate function is nil") } definitionSnippet, err := buildAllDefinitions(evaluate, framework.Definitions) if err != nil { @@ -38,95 +36,87 @@ func NewEvaluator(framework *model.Framework, evaluate model.Evaluate) (*Evaluat } // Evaluate evaluates snippet for a Resource data -func (e *Evaluator) Evaluate(resourceData *model.Data) (string, model.Error) { - const defaultErrKey = "Evaluate" +func (e *Evaluator) Evaluate(resourceData *model.Data) (bool, error) { if resourceData == nil { - return model.SkipNullValue, model.BuildError(defaultErrKey, errors.New("resource data is nil")) + return false, errors.New("resource data is nil") } resourceSnippet := string(resourceData.Content) previousOutputSnippet := model.SkipNullValue - for _, procedure := range e.framework.Procedures { + for i, procedure := range e.framework.Procedures { + if procedure == nil { + return false, fmt.Errorf("procedure [%d] is nil", i) + } snippet, err := buildSnippet(resourceSnippet, e.definitionSnippet, previousOutputSnippet, procedure) if err != nil { - return model.SkipNullValue, err + return false, err } result, evalErr := e.evaluate(procedure.Name, snippet) if evalErr != nil { - return model.SkipNullValue, model.BuildError(defaultErrKey, evalErr) + return false, evalErr } if model.IsSkipResult[result] { previousOutputSnippet = model.SkipNullValue } else { - evalResult := e.resultToError(result) - if procedure.OutputIsError { - return model.SkipNullValue, evalResult + success, err := treatOutput( + &model.Data{ + Type: resourceData.Type, + Path: resourceData.Path, + Content: []byte(result), + }, + procedure.Output, + ) + if err != nil { + return false, err + } + if !success { + return false, nil } previousOutputSnippet = result } } - return previousOutputSnippet, nil -} - -func (e *Evaluator) resultToError(result string) model.Error { - const defaultErrKey = "evaluation result" - fn, err := endec.Decodes.Get(jsonFormat) - if err != nil { - return err - } - var tmp interface{} - if err = fn([]byte(result), &tmp); err != nil { - return model.BuildError(defaultErrKey, errors.New(result)) - } - return model.BuildError(defaultErrKey, tmp) + return true, nil } -func buildAllDefinitions(evaluate model.Evaluate, definitions []*model.Definition) (string, model.Error) { - const defaultErrKey = "buildAllDefinitions" - +func buildAllDefinitions(evaluate model.Evaluate, definitions []*model.Definition) (string, error) { wg := &sync.WaitGroup{} mtx := &sync.Mutex{} nameToSnippet := make(map[string]string) - outputError := make(model.Error) + outputError := &model.Error{} for i, def := range definitions { wg.Add(1) - go func(idx int, e model.Evaluate, w *sync.WaitGroup, m *sync.Mutex, d *model.Definition) { + go func(idx int, w *sync.WaitGroup, m *sync.Mutex, d *model.Definition) { defer w.Done() - - defSnippet, err := buildOneDefinition(e, d) + defSnippet, err := buildOneDefinition(evaluate, d) if err != nil { - key := fmt.Sprintf("%s [%d]", defaultErrKey, idx) + key := fmt.Sprintf("%d", idx) if d != nil { - key = fmt.Sprintf("%s [%s]", defaultErrKey, d.Name) + key = d.Name } - m.Lock() - outputError[key] = err - m.Unlock() + outputError.Add(key, err) } else { m.Lock() nameToSnippet[d.Name] = defSnippet m.Unlock() } - }(i, evaluate, wg, mtx, def) - + }(i, wg, mtx, def) } wg.Wait() - if len(outputError) > 0 { + if outputError.Length() > 0 { return model.SkipNullValue, outputError } var outputSnippets []string for key, value := range nameToSnippet { - outputSnippets = append(outputSnippets, fmt.Sprintf(`"%s": %s`, key, value)) + outputSnippets = append(outputSnippets, fmt.Sprintf(`"%s": %s,`, key, value)) } return fmt.Sprintf("{%s}", strings.Join(outputSnippets, "\n")), nil } -func buildOneDefinition(evaluate model.Evaluate, definition *model.Definition) (string, model.Error) { - const defaultErrKey = "buildOneDefinition" +func buildOneDefinition(evaluate model.Evaluate, definition *model.Definition) (string, error) { if definition == nil { - return model.SkipNullValue, model.BuildError(defaultErrKey, errors.New("definition is nil")) + return model.SkipNullValue, errors.New("definition is nil") } var defData string for i, data := range definition.ListOfData { @@ -197,7 +187,7 @@ construct (definition) ) result, err := evaluate(definition.Name, defSnippet) if err != nil { - return model.SkipNullValue, model.BuildError(defaultErrKey, err) + return model.SkipNullValue, err } defSnippet = result } @@ -209,13 +199,9 @@ func buildSnippet( definitionSnippet string, previousOutputSnippet string, procedure *model.Procedure, -) (string, model.Error) { - const defaultErrKey = "buildSnippet" - if procedure == nil { - return model.SkipNullValue, model.BuildError(defaultErrKey, errors.New("procedure is nil")) - } +) (string, error) { if procedure.Data == nil { - return model.SkipNullValue, model.BuildError(defaultErrKey, errors.New("procedure data is nil")) + return model.SkipNullValue, fmt.Errorf("procedure data for [%s] is nil", procedure.Name) } output := fmt.Sprintf(` /* diff --git a/core/evaluator_test.go b/core/evaluator_test.go index 61bf479..410e821 100644 --- a/core/evaluator_test.go +++ b/core/evaluator_test.go @@ -17,7 +17,7 @@ type EvaluatorSuite struct { } func (e *EvaluatorSuite) TestEvaluate() { - e.Run("should return null and error if resource data is nil", func() { + e.Run("should return false and error if resource data is nil", func() { framework := &model.Framework{} var resourceData *model.Data = nil var evaluate model.Evaluate = func(name, snippet string) (string, error) { @@ -25,7 +25,7 @@ func (e *EvaluatorSuite) TestEvaluate() { } evaluator, _ := core.NewEvaluator(framework, evaluate) - expectedValue := model.SkipNullValue + expectedValue := false actualValue, actualErr := evaluator.Evaluate(resourceData) @@ -33,7 +33,7 @@ func (e *EvaluatorSuite) TestEvaluate() { e.NotNil(actualErr) }) - e.Run("should return null and error if one or more procedures are nil", func() { + e.Run("should return false and error if one or more procedures are nil", func() { framework := &model.Framework{ Procedures: []*model.Procedure{nil}, } @@ -43,7 +43,7 @@ func (e *EvaluatorSuite) TestEvaluate() { } evaluator, _ := core.NewEvaluator(framework, evaluate) - expectedValue := model.SkipNullValue + expectedValue := false actualValue, actualErr := evaluator.Evaluate(resourceData) @@ -65,7 +65,7 @@ func (e *EvaluatorSuite) TestEvaluate() { } evaluator, _ := core.NewEvaluator(framework, evaluate) - expectedValue := model.SkipNullValue + expectedValue := false actualValue, actualErr := evaluator.Evaluate(resourceData) @@ -90,7 +90,7 @@ func (e *EvaluatorSuite) TestEvaluate() { } evaluator, _ := core.NewEvaluator(framework, evaluate) - expectedValue := model.SkipNullValue + expectedValue := false actualValue, actualErr := evaluator.Evaluate(resourceData) @@ -98,59 +98,7 @@ func (e *EvaluatorSuite) TestEvaluate() { e.NotNil(actualErr) }) - e.Run("should return null and unformatted error if evaluation returns random result but considered error", func() { - framework := &model.Framework{ - Procedures: []*model.Procedure{ - { - Name: "procecure_test", - Data: &model.Data{ - Content: []byte("test content"), - }, - OutputIsError: true, - }, - }, - } - var resourceData *model.Data = &model.Data{} - var evaluate model.Evaluate = func(name, snippet string) (string, error) { - return "test result", nil - } - evaluator, _ := core.NewEvaluator(framework, evaluate) - - expectedValue := model.SkipNullValue - - actualValue, actualErr := evaluator.Evaluate(resourceData) - - e.Equal(expectedValue, actualValue) - e.NotNil(actualErr) - }) - - e.Run("should return null and formatted error if evaluation returns json result but considered error", func() { - framework := &model.Framework{ - Procedures: []*model.Procedure{ - { - Name: "procecure_test", - Data: &model.Data{ - Content: []byte("test content"), - }, - OutputIsError: true, - }, - }, - } - var resourceData *model.Data = &model.Data{} - var evaluate model.Evaluate = func(name, snippet string) (string, error) { - return "{\"message\": \"error\"}", nil - } - evaluator, _ := core.NewEvaluator(framework, evaluate) - - expectedValue := model.SkipNullValue - - actualValue, actualErr := evaluator.Evaluate(resourceData) - - e.Equal(expectedValue, actualValue) - e.NotNil(actualErr) - }) - - e.Run("should return result and nil if no error is encountered", func() { + e.Run("should return true and nil if no error is encountered", func() { framework := &model.Framework{ Procedures: []*model.Procedure{ { @@ -167,7 +115,7 @@ func (e *EvaluatorSuite) TestEvaluate() { } evaluator, _ := core.NewEvaluator(framework, evaluate) - expectedValue := "{\"message\": \"error\"}" + expectedValue := true actualValue, actualErr := evaluator.Evaluate(resourceData) diff --git a/core/explorer.go b/core/explorer.go new file mode 100644 index 0000000..0686bfb --- /dev/null +++ b/core/explorer.go @@ -0,0 +1,18 @@ +package core + +import ( + "strings" + + "github.com/gojek/optimus-extension-valor/registry/explorer" +) + +// ExplorePaths explores the given root path for the type and format +func ExplorePaths(rootPath, _type, format string) ([]string, error) { + exPath, err := explorer.Explorers.Get(_type) + if err != nil { + return nil, err + } + return exPath(rootPath, func(path string) bool { + return strings.HasSuffix(path, format) + }) +} diff --git a/core/loader.go b/core/loader.go index 09493b2..6635698 100644 --- a/core/loader.go +++ b/core/loader.go @@ -3,7 +3,6 @@ package core import ( "errors" "fmt" - "strings" "sync" "github.com/gojek/optimus-extension-valor/model" @@ -16,30 +15,11 @@ import ( type Loader struct { } -// LoadResource loads a Resource based on its recipe -func (l *Loader) LoadResource(rcp *recipe.Resource) (*model.Resource, model.Error) { - const defaultErrKey = "LoadResource" +// LoadFramework loads framework based on the specified recipe +func (l *Loader) LoadFramework(rcp *recipe.Framework) (*model.Framework, error) { if rcp == nil { - return nil, model.BuildError(defaultErrKey, errors.New("resource recipe is nil")) + return nil, errors.New("framework recipe is nil") } - listOfData, err := l.loadAllData(rcp.Path, rcp.Type, rcp.Format) - if err != nil { - return nil, err - } - return &model.Resource{ - Name: rcp.Name, - ListOfData: listOfData, - FrameworkNames: rcp.FrameworkNames, - }, nil -} - -// LoadFramework loads a framework based on its recipe -func (l *Loader) LoadFramework(rcp *recipe.Framework) (*model.Framework, model.Error) { - const defaultErrKey = "LoadFramework" - if rcp == nil { - return nil, model.BuildError(defaultErrKey, errors.New("framework recipe is nil")) - } - definitions, defError := l.loadAllDefinitions(rcp.Definitions) if defError != nil { return nil, defError @@ -53,82 +33,87 @@ func (l *Loader) LoadFramework(rcp *recipe.Framework) (*model.Framework, model.E return nil, proError } return &model.Framework{ - Name: rcp.Name, - Definitions: definitions, - Schemas: schemas, - Procedures: procedures, - OutputTargets: l.convertOutputTargets(rcp.OutputTargets), + Name: rcp.Name, + Definitions: definitions, + Schemas: schemas, + Procedures: procedures, }, nil } -func (l *Loader) convertOutputTargets(targets []*recipe.OutputTarget) []*model.OutputTarget { - output := make([]*model.OutputTarget, len(targets)) - for i, t := range targets { - output[i] = &model.OutputTarget{ - Name: t.Name, - Format: t.Format, - Type: t.Type, - Path: t.Path, - } - } - return output -} - -func (l *Loader) loadAllDefinitions(rcps []*recipe.Definition) ([]*model.Definition, model.Error) { - const defaultErrKey = "loadAllDefinitions" - +func (l *Loader) loadAllProcedures(rcps []*recipe.Procedure) ([]*model.Procedure, error) { wg := &sync.WaitGroup{} mtx := &sync.Mutex{} - outputData := make([]*model.Definition, len(rcps)) - outputError := make(model.Error) + outputData := make([]*model.Procedure, len(rcps)) + outputError := &model.Error{} for i, rcp := range rcps { wg.Add(1) - - go func(idx int, w *sync.WaitGroup, m *sync.Mutex, r *recipe.Definition) { + go func(idx int, w *sync.WaitGroup, m *sync.Mutex, r *recipe.Procedure) { defer wg.Done() - definition, err := l.LoadDefinition(r) + procedure, err := l.LoadProcedure(r) if err != nil { - key := fmt.Sprintf("%s [%d]", defaultErrKey, idx) - m.Lock() - outputError[key] = err - m.Unlock() + key := fmt.Sprintf("%d", idx) + if r != nil { + key = r.Name + } + outputError.Add(key, err) } else { m.Lock() - outputData[idx] = definition + outputData[idx] = procedure m.Unlock() } }(i, wg, mtx, rcp) } wg.Wait() - if len(outputError) > 0 { + if outputError.Length() > 0 { return nil, outputError } return outputData, nil } -func (l *Loader) loadAllSchemas(rcps []*recipe.Schema) ([]*model.Schema, model.Error) { - const defaultErrKey = "loadAllSchemas" +// LoadProcedure loads procedure based on the specified recipe +func (l *Loader) LoadProcedure(rcp *recipe.Procedure) (*model.Procedure, error) { + if rcp == nil { + return nil, errors.New("procedure recipe is nil") + } + paths, err := ExplorePaths(rcp.Path, rcp.Type, jsonnetFormat) + if err != nil { + return nil, err + } + if len(paths) == 0 { + return nil, fmt.Errorf("[%s] procedure for recipe [%s] cannot be found", jsonnetFormat, rcp.Name) + } + data, err := l.LoadData(paths[0], rcp.Type, jsonnetFormat) + if err != nil { + return nil, err + } + return &model.Procedure{ + Name: rcp.Name, + Data: data, + Output: l.convertOutput(rcp.Output), + }, nil +} +func (l *Loader) loadAllSchemas(rcps []*recipe.Schema) ([]*model.Schema, error) { wg := &sync.WaitGroup{} mtx := &sync.Mutex{} outputData := make([]*model.Schema, len(rcps)) - outputError := make(model.Error) + outputError := &model.Error{} for i, rcp := range rcps { wg.Add(1) go func(idx int, w *sync.WaitGroup, m *sync.Mutex, r *recipe.Schema) { defer wg.Done() - schema, err := l.LoadSchema(r) if err != nil { - key := fmt.Sprintf("%s [%d]", defaultErrKey, idx) - m.Lock() - outputError[key] = err - m.Unlock() + key := fmt.Sprintf("%d", idx) + if r != nil { + key = r.Name + } + outputError.Add(key, err) } else { m.Lock() outputData[idx] = schema @@ -138,60 +123,104 @@ func (l *Loader) loadAllSchemas(rcps []*recipe.Schema) ([]*model.Schema, model.E } wg.Wait() - if len(outputError) > 0 { + if outputError.Length() > 0 { return nil, outputError } return outputData, nil } -func (l *Loader) loadAllProcedures(rcps []*recipe.Procedure) ([]*model.Procedure, model.Error) { - const defaultErrKey = "loadAllProcedures" +// LoadSchema loads schema based on the specified recipe +func (l *Loader) LoadSchema(rcp *recipe.Schema) (*model.Schema, error) { + if rcp == nil { + return nil, errors.New("schema recipe is nil") + } + paths, err := ExplorePaths(rcp.Path, rcp.Type, jsonFormat) + if err != nil { + return nil, err + } + if len(paths) == 0 { + return nil, fmt.Errorf("[%s] schema for recipe [%s] cannot be found", jsonFormat, rcp.Name) + } + data, err := l.LoadData(paths[0], rcp.Type, jsonFormat) + return &model.Schema{ + Name: rcp.Name, + Data: data, + Output: l.convertOutput(rcp.Output), + }, nil +} +func (l *Loader) convertOutput(output *recipe.Output) *model.Output { + if output == nil { + return nil + } + targets := make([]*model.Target, len(output.Targets)) + for i, t := range output.Targets { + targets[i] = &model.Target{ + Name: t.Name, + Format: t.Format, + Type: t.Type, + Path: t.Path, + } + } + return &model.Output{ + TreatAs: model.OutputTreatment(output.TreatAs), + Targets: targets, + } +} + +func (l *Loader) loadAllDefinitions(rcps []*recipe.Definition) ([]*model.Definition, error) { wg := &sync.WaitGroup{} mtx := &sync.Mutex{} - outputData := make([]*model.Procedure, len(rcps)) - outputError := make(model.Error) + outputData := make([]*model.Definition, len(rcps)) + outputError := &model.Error{} for i, rcp := range rcps { wg.Add(1) - go func(idx int, w *sync.WaitGroup, m *sync.Mutex, r *recipe.Procedure) { + go func(idx int, w *sync.WaitGroup, m *sync.Mutex, r *recipe.Definition) { defer wg.Done() - - procedure, err := l.LoadProcedure(r) + definition, err := l.LoadDefinition(r) if err != nil { - key := fmt.Sprintf("%s [%d]", defaultErrKey, idx) - m.Lock() - outputError[key] = err - m.Unlock() + key := fmt.Sprintf("%d", idx) + if r != nil { + key = r.Name + } + outputError.Add(key, err) } else { m.Lock() - outputData[idx] = procedure + outputData[idx] = definition m.Unlock() } }(i, wg, mtx, rcp) } wg.Wait() - if len(outputError) > 0 { + if outputError.Length() > 0 { return nil, outputError } return outputData, nil } -// LoadDefinition loads definition based on its recipe -func (l *Loader) LoadDefinition(rcp *recipe.Definition) (*model.Definition, model.Error) { - const defaultErrKey = "LoadDefinition" +// LoadDefinition loads definition based on the specified recipe +func (l *Loader) LoadDefinition(rcp *recipe.Definition) (*model.Definition, error) { if rcp == nil { - return nil, model.BuildError(defaultErrKey, errors.New("definition recipe is nil")) + return nil, errors.New("definition recipe is nil") } - listOfData, err := l.loadAllData(rcp.Path, rcp.Type, rcp.Format) + paths, err := ExplorePaths(rcp.Path, rcp.Type, rcp.Format) if err != nil { return nil, err } + listOfData := make([]*model.Data, len(paths)) + for i, p := range paths { + data, err := l.LoadData(p, rcp.Type, rcp.Format) + if err != nil { + return nil, err + } + listOfData[i] = data + } var functionData *model.Data if rcp.Function != nil { - data, err := l.loadOneData(rcp.Function.Path, rcp.Function.Type, jsonnetFormat) + data, err := l.LoadData(rcp.Function.Path, rcp.Function.Type, jsonnetFormat) if err != nil { return nil, err } @@ -204,103 +233,42 @@ func (l *Loader) LoadDefinition(rcp *recipe.Definition) (*model.Definition, mode }, nil } -// LoadSchema loads schema based on its recipe -func (l *Loader) LoadSchema(rcp *recipe.Schema) (*model.Schema, model.Error) { - const defaultErrKey = "LoadSchema" - if rcp == nil { - return nil, model.BuildError(defaultErrKey, errors.New("schema recipe is nil")) - } - data, err := l.loadOneData(rcp.Path, rcp.Type, jsonFormat) +// LoadData loads data based on the specified path, type, and format +func (l *Loader) LoadData(path, _type, format string) (*model.Data, error) { + reader, err := l.getReader(path, _type, format) if err != nil { return nil, err } - return &model.Schema{ - Name: rcp.Name, - Data: data, - }, nil + return reader.Read() } -// LoadProcedure loads procedure based on its recipe -func (l *Loader) LoadProcedure(rcp *recipe.Procedure) (*model.Procedure, model.Error) { - const defaultErrKey = "LoadProcedure" - if rcp == nil { - return nil, model.BuildError(defaultErrKey, errors.New("procedure recipe is nil")) - } - data, err := l.loadOneData(rcp.Path, rcp.Type, jsonnetFormat) - if err != nil { - return nil, err - } - return &model.Procedure{ - Name: rcp.Name, - OutputIsError: rcp.OutputIsError, - Data: data, - }, nil -} - -func (l *Loader) loadOneData(path, _type, format string) (*model.Data, model.Error) { - reader, err := l.getLoadReader(path, _type, format) - if err != nil { - return nil, err - } - return reader.ReadOne() -} - -func (l *Loader) loadAllData(path, _type, format string) ([]*model.Data, model.Error) { - const defaultErrKey = "loadAllData" - reader, err := l.getLoadReader(path, _type, format) - if err != nil { - return nil, err - } - data, err := reader.ReadAll() - if err != nil { - return nil, err - } - return data, nil -} - -func (l *Loader) getLoadReader(path, _type, format string) (model.Reader, model.Error) { - const defaultErrKey = "getLoadReader" +func (l *Loader) getReader(path, _type, format string) (model.Reader, error) { readerFn, err := io.Readers.Get(_type) if err != nil { return nil, err } reader := readerFn( - l.getPath(path), - l.filterPath(format), - l.postProcess(_type, format), + func() string { + return path + }, + func(path string, content []byte) (*model.Data, error) { + if !skipReformat[format] { + fn, err := formatter.Formats.Get(format, jsonFormat) + if err != nil { + return nil, err + } + reformattedContent, err := fn(content) + if err != nil { + return nil, err + } + content = reformattedContent + } + return &model.Data{ + Path: path, + Type: _type, + Content: content, + }, nil + }, ) return reader, nil } - -func (l *Loader) getPath(path string) model.GetPath { - return func() string { - return path - } -} - -func (l *Loader) filterPath(suffix string) model.FilterPath { - return func(path string) bool { - return strings.HasSuffix(path, suffix) - } -} - -func (l *Loader) postProcess(_type, format string) model.PostProcess { - return func(path string, content []byte) (*model.Data, model.Error) { - if !skipReformat[format] { - fn, err := formatter.Formats.Get(format, jsonFormat) - if err != nil { - return nil, err - } - reformattedContent, err := fn(content) - if err != nil { - return nil, err - } - content = reformattedContent - } - return &model.Data{ - Path: path, - Type: _type, - Content: content, - }, nil - } -} diff --git a/core/loader_test.go b/core/loader_test.go index fa238a7..c9675e8 100644 --- a/core/loader_test.go +++ b/core/loader_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/gojek/optimus-extension-valor/core" + _ "github.com/gojek/optimus-extension-valor/plugin/explorer" _ "github.com/gojek/optimus-extension-valor/plugin/formatter" _ "github.com/gojek/optimus-extension-valor/plugin/io" "github.com/gojek/optimus-extension-valor/recipe" @@ -15,10 +16,15 @@ import ( ) const ( - defaulDirName = "./out" - defaultValidFileName = "valor.yaml" - defaultInvalidFileName = "valor.invalid" - defaultContent = "message: 0" + defaultDirName = "./out" + defaultSchemaFileName = "schema.json" + defaultSchemaContent = "{\"message\":0}" + defaultDefinitionFileName = "definition.json" + defaultDefinitionContent = "{\"message\":0}" + defaultDefinitionFormat = "json" + defaultProcedureFileName = "procedure.jsonnet" + defaultProcedureContent = "{\"message\":0}" + defaultValidType = "file" ) type LoaderSuite struct { @@ -26,218 +32,180 @@ type LoaderSuite struct { } func (l *LoaderSuite) SetupSuite() { - if err := os.MkdirAll(defaulDirName, os.ModePerm); err != nil { + if err := os.MkdirAll(defaultDirName, os.ModePerm); err != nil { panic(err) } - filePath := path.Join(defaulDirName, defaultValidFileName) - if err := ioutil.WriteFile(filePath, []byte(defaultContent), os.ModePerm); err != nil { + filePath := path.Join(defaultDirName, defaultSchemaFileName) + if err := ioutil.WriteFile(filePath, []byte(defaultSchemaContent), os.ModePerm); err != nil { panic(err) } - filePath = path.Join(defaulDirName, defaultInvalidFileName) - if err := ioutil.WriteFile(filePath, []byte(defaultContent), os.ModePerm); err != nil { + filePath = path.Join(defaultDirName, defaultDefinitionFileName) + if err := ioutil.WriteFile(filePath, []byte(defaultDefinitionContent), os.ModePerm); err != nil { + panic(err) + } + filePath = path.Join(defaultDirName, defaultProcedureFileName) + if err := ioutil.WriteFile(filePath, []byte(defaultProcedureContent), os.ModePerm); err != nil { panic(err) } } -func (l *LoaderSuite) TestLoadResource() { +func (l *LoaderSuite) TestLoadFramework() { l.Run("should return nil and error if recipe is nil", func() { - var rcp *recipe.Resource = nil + var rcp *recipe.Framework = nil loader := &core.Loader{} - actualValue, actualErr := loader.LoadResource(rcp) + actualValue, actualErr := loader.LoadFramework(rcp) l.Nil(actualValue) l.NotNil(actualErr) }) - l.Run("should return nil and error if recipe contains invalid type", func() { - rcp := &recipe.Resource{ - Name: "test_resource", - Type: "invalid_type", - Format: "yaml", - Path: path.Join(defaulDirName, defaultValidFileName), - FrameworkNames: []string{ - "framework_target", + l.Run("should return nil and error if error during loading definition", func() { + rcp := &recipe.Framework{ + Name: "test_framework", + Definitions: []*recipe.Definition{ + { + Name: "test_definition", + Format: defaultDefinitionFormat, + Type: defaultValidType, + Path: path.Join(defaultDirName, defaultDefinitionFileName), + }, + nil, }, } loader := &core.Loader{} - actualValue, actualErr := loader.LoadResource(rcp) + actualValue, actualErr := loader.LoadFramework(rcp) l.Nil(actualValue) l.NotNil(actualErr) }) - l.Run("should return nil and error if recipe contains invalid format", func() { - rcp := &recipe.Resource{ - Name: "test_resource", - Type: "file", - Format: "invalid_format", - Path: path.Join(defaulDirName, defaultValidFileName), - FrameworkNames: []string{ - "framework_target", + l.Run("should return nil and error if error during loading schema", func() { + rcp := &recipe.Framework{ + Name: "test_framework", + Definitions: []*recipe.Definition{ + { + Name: "test_definition", + Format: defaultDefinitionFormat, + Type: defaultValidType, + Path: path.Join(defaultDirName, defaultDefinitionFileName), + }, }, - } - loader := &core.Loader{} - - actualValue, actualErr := loader.LoadResource(rcp) - - l.Nil(actualValue) - l.NotNil(actualErr) - }) - - l.Run("should return nil and error if recipe contains inconsistent format", func() { - rcp := &recipe.Resource{ - Name: "test_resource", - Type: "dir", - Format: "inconsistent", - Path: defaulDirName, - FrameworkNames: []string{ - "framework_target", + Schemas: []*recipe.Schema{ + { + Name: "test_schema", + Type: defaultValidType, + Path: path.Join(defaultDirName, defaultSchemaFileName), + }, + nil, }, } loader := &core.Loader{} - actualValue, actualErr := loader.LoadResource(rcp) + actualValue, actualErr := loader.LoadFramework(rcp) l.Nil(actualValue) l.NotNil(actualErr) }) - l.Run("should return nil and error if recipe contains invalid path", func() { - rcp := &recipe.Resource{ - Name: "test_resource", - Type: "file", - Format: "yaml", - Path: defaulDirName, - FrameworkNames: []string{ - "framework_target", + l.Run("should return nil and error if error during loading procedure", func() { + rcp := &recipe.Framework{ + Name: "test_framework", + Definitions: []*recipe.Definition{ + { + Name: "test_definition", + Format: defaultDefinitionFormat, + Type: defaultValidType, + Path: path.Join(defaultDirName, defaultDefinitionFileName), + }, + }, + Schemas: []*recipe.Schema{ + { + Name: "test_schema", + Type: defaultValidType, + Path: path.Join(defaultDirName, defaultSchemaFileName), + }, + }, + Procedures: []*recipe.Procedure{ + { + Name: "test_procedure", + Type: defaultValidType, + Path: path.Join(defaultDirName, defaultProcedureFileName), + }, + nil, }, } loader := &core.Loader{} - actualValue, actualErr := loader.LoadResource(rcp) + actualValue, actualErr := loader.LoadFramework(rcp) l.Nil(actualValue) l.NotNil(actualErr) }) l.Run("should return value and nil if no error is encountered", func() { - rcp := &recipe.Resource{ - Name: "test_resource", - Type: "file", - Format: "yaml", - Path: path.Join(defaulDirName, defaultValidFileName), - FrameworkNames: []string{ - "framework_target", + rcp := &recipe.Framework{ + Name: "test_framework", + Definitions: []*recipe.Definition{ + { + Name: "test_definition", + Format: defaultDefinitionFormat, + Type: defaultValidType, + Path: path.Join(defaultDirName, defaultDefinitionFileName), + }, + }, + Schemas: []*recipe.Schema{ + { + Name: "test_schema", + Type: defaultValidType, + Path: path.Join(defaultDirName, defaultSchemaFileName), + }, }, } loader := &core.Loader{} - actualValue, actualErr := loader.LoadResource(rcp) + actualValue, actualErr := loader.LoadFramework(rcp) l.NotNil(actualValue) l.Nil(actualErr) }) } -func (l *LoaderSuite) TestLoadDefinition() { +func (l *LoaderSuite) TestLoadProcedure() { l.Run("should return nil and error if recipe is nil", func() { - var rcp *recipe.Definition = nil + var rcp *recipe.Procedure = nil loader := &core.Loader{} - actualValue, actualErr := loader.LoadDefinition(rcp) + actualValue, actualErr := loader.LoadProcedure(rcp) l.Nil(actualValue) l.NotNil(actualErr) }) l.Run("should return nil and error if recipe contains invalid type", func() { - rcp := &recipe.Definition{ - Name: "test_definition", - Type: "invalid_type", - Format: "yaml", - Path: path.Join(defaulDirName, defaultValidFileName), - } - loader := &core.Loader{} - - actualValue, actualErr := loader.LoadDefinition(rcp) - - l.Nil(actualValue) - l.NotNil(actualErr) - }) - - l.Run("should return nil and error if recipe contains invalid format", func() { - rcp := &recipe.Definition{ - Name: "test_definition", - Type: "file", - Format: "invalid_format", - Path: path.Join(defaulDirName, defaultValidFileName), - } - loader := &core.Loader{} - - actualValue, actualErr := loader.LoadDefinition(rcp) - - l.Nil(actualValue) - l.NotNil(actualErr) - }) - - l.Run("should return nil and error if recipe contains inconsistent format", func() { - rcp := &recipe.Definition{ - Name: "test_definition", - Type: "file", - Format: "inconsistent", - Path: path.Join(defaulDirName, defaultInvalidFileName), - } - loader := &core.Loader{} - - actualValue, actualErr := loader.LoadDefinition(rcp) - - l.Nil(actualValue) - l.NotNil(actualErr) - }) - - l.Run("should return nil and error if recipe contains invalid path", func() { - rcp := &recipe.Definition{ - Name: "test_definition", - Type: "file", - Format: "yaml", - Path: defaulDirName, - } - loader := &core.Loader{} - - actualValue, actualErr := loader.LoadDefinition(rcp) - - l.Nil(actualValue) - l.NotNil(actualErr) - }) - - l.Run("should return nil and error if recipe function is empty", func() { - rcp := &recipe.Definition{ - Name: "test_definition", - Type: "file", - Format: "yaml", - Path: path.Join(defaulDirName, defaultValidFileName), - Function: &recipe.Function{}, + rcp := &recipe.Procedure{ + Name: "test_procedure", + Type: "invalid_type", + Path: path.Join(defaultDirName, defaultProcedureFileName), } loader := &core.Loader{} - actualValue, actualErr := loader.LoadDefinition(rcp) + actualValue, actualErr := loader.LoadProcedure(rcp) l.Nil(actualValue) l.NotNil(actualErr) }) l.Run("should return value and nil if no error is encountered", func() { - rcp := &recipe.Definition{ - Name: "test_definition", - Type: "file", - Format: "yaml", - Path: path.Join(defaulDirName, defaultValidFileName), + rcp := &recipe.Procedure{ + Name: "test_procedure", + Type: defaultValidType, + Path: path.Join(defaultDirName, defaultProcedureFileName), } loader := &core.Loader{} - actualValue, actualErr := loader.LoadDefinition(rcp) + actualValue, actualErr := loader.LoadProcedure(rcp) l.NotNil(actualValue) l.Nil(actualErr) @@ -259,35 +227,7 @@ func (l *LoaderSuite) TestLoadSchema() { rcp := &recipe.Schema{ Name: "test_schema", Type: "invalid_type", - Path: path.Join(defaulDirName, defaultValidFileName), - } - loader := &core.Loader{} - - actualValue, actualErr := loader.LoadSchema(rcp) - - l.Nil(actualValue) - l.NotNil(actualErr) - }) - - l.Run("should return nil and error if recipe contains inconsistent format", func() { - rcp := &recipe.Schema{ - Name: "test_schema", - Type: "dir", - Path: defaulDirName, - } - loader := &core.Loader{} - - actualValue, actualErr := loader.LoadSchema(rcp) - - l.Nil(actualValue) - l.NotNil(actualErr) - }) - - l.Run("should return nil and error if recipe contains invalid path", func() { - rcp := &recipe.Schema{ - Name: "test_schema", - Type: "file", - Path: defaulDirName, + Path: path.Join(defaultDirName, defaultSchemaFileName), } loader := &core.Loader{} @@ -300,8 +240,8 @@ func (l *LoaderSuite) TestLoadSchema() { l.Run("should return value and nil if no error is encountered", func() { rcp := &recipe.Schema{ Name: "test_schema", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), + Type: defaultValidType, + Path: path.Join(defaultDirName, defaultSchemaFileName), } loader := &core.Loader{} @@ -312,214 +252,104 @@ func (l *LoaderSuite) TestLoadSchema() { }) } -func (l *LoaderSuite) TestLoadProcedure() { +func (l *LoaderSuite) TestLoadDefinition() { l.Run("should return nil and error if recipe is nil", func() { - var rcp *recipe.Procedure = nil + var rcp *recipe.Definition = nil loader := &core.Loader{} - actualValue, actualErr := loader.LoadProcedure(rcp) + actualValue, actualErr := loader.LoadDefinition(rcp) l.Nil(actualValue) l.NotNil(actualErr) }) l.Run("should return nil and error if recipe contains invalid type", func() { - rcp := &recipe.Procedure{ - Name: "test_procedure", - Type: "invalid_type", - Path: path.Join(defaulDirName, defaultValidFileName), - } - loader := &core.Loader{} - - actualValue, actualErr := loader.LoadProcedure(rcp) - - l.Nil(actualValue) - l.NotNil(actualErr) - }) - - l.Run("should return nil and error if recipe contains inconsistent format", func() { - rcp := &recipe.Procedure{ - Name: "test_procedure", - Type: "dir", - Path: defaulDirName, + rcp := &recipe.Definition{ + Name: "test_definition", + Type: "invalid_type", + Format: defaultDefinitionFormat, + Path: path.Join(defaultDirName, defaultDefinitionFileName), } loader := &core.Loader{} - actualValue, actualErr := loader.LoadProcedure(rcp) + actualValue, actualErr := loader.LoadDefinition(rcp) l.Nil(actualValue) l.NotNil(actualErr) }) - l.Run("should return nil and error if recipe contains invalid path", func() { - rcp := &recipe.Procedure{ - Name: "test_procedure", - Type: "file", - Path: defaulDirName, + l.Run("should return nil and error if recipe function is set but empty", func() { + rcp := &recipe.Definition{ + Name: "test_definition", + Type: defaultValidType, + Format: defaultDefinitionFormat, + Path: path.Join(defaultDirName, defaultDefinitionFileName), + Function: &recipe.Function{}, } loader := &core.Loader{} - actualValue, actualErr := loader.LoadProcedure(rcp) + actualValue, actualErr := loader.LoadDefinition(rcp) l.Nil(actualValue) l.NotNil(actualErr) }) l.Run("should return value and nil if no error is encountered", func() { - rcp := &recipe.Procedure{ - Name: "test_procedure", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), + rcp := &recipe.Definition{ + Name: "test_definition", + Type: defaultValidType, + Format: defaultDefinitionFormat, + Path: path.Join(defaultDirName, defaultDefinitionFileName), } loader := &core.Loader{} - actualValue, actualErr := loader.LoadProcedure(rcp) + actualValue, actualErr := loader.LoadDefinition(rcp) l.NotNil(actualValue) l.Nil(actualErr) }) } -func (l *LoaderSuite) TestLoadFramework() { - l.Run("should return nil and error if recipe is nil", func() { - var rcp *recipe.Framework = nil - loader := &core.Loader{} - - actualValue, actualErr := loader.LoadFramework(rcp) - - l.Nil(actualValue) - l.NotNil(actualErr) - }) - - l.Run("should return nil and error if error during loading definition", func() { - rcp := &recipe.Framework{ - Name: "test_framework", - Definitions: []*recipe.Definition{ - { - Name: "test_definition", - Format: "yaml", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), - }, - nil, - }, - } - loader := &core.Loader{} - - actualValue, actualErr := loader.LoadFramework(rcp) - - l.Nil(actualValue) - l.NotNil(actualErr) - }) - - l.Run("should return nil and error if error during loading schema", func() { - rcp := &recipe.Framework{ - Name: "test_framework", - Definitions: []*recipe.Definition{ - { - Name: "test_definition", - Format: "yaml", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), - }, - }, - Schemas: []*recipe.Schema{ - { - Name: "test_schema", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), - }, - nil, - }, - } +func (l *LoaderSuite) TestLoadData() { + l.Run("should return nil and error if type is invalid", func() { loader := &core.Loader{} + pt := path.Join(defaultDirName, defaultDefinitionFileName) + _type := "invalid_type" + format := defaultDefinitionFormat - actualValue, actualErr := loader.LoadFramework(rcp) + actualData, actualErr := loader.LoadData(pt, _type, format) - l.Nil(actualValue) + l.Nil(actualData) l.NotNil(actualErr) }) - l.Run("should return nil and error if error during loading procedure", func() { - rcp := &recipe.Framework{ - Name: "test_framework", - Definitions: []*recipe.Definition{ - { - Name: "test_definition", - Format: "yaml", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), - }, - }, - Schemas: []*recipe.Schema{ - { - Name: "test_schema", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), - }, - }, - Procedures: []*recipe.Procedure{ - { - Name: "test_procedure", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), - }, - nil, - }, - } + l.Run("should return nil and error if format is invalid", func() { loader := &core.Loader{} + pt := path.Join(defaultDirName, defaultDefinitionFileName) + _type := defaultValidType + format := "invalid_format" - actualValue, actualErr := loader.LoadFramework(rcp) + actualData, actualErr := loader.LoadData(pt, _type, format) - l.Nil(actualValue) + l.Nil(actualData) l.NotNil(actualErr) }) - l.Run("should return value and nil if no error is encountered", func() { - rcp := &recipe.Framework{ - Name: "test_framework", - Definitions: []*recipe.Definition{ - { - Name: "test_definition", - Format: "yaml", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), - }, - }, - Schemas: []*recipe.Schema{ - { - Name: "test_schema", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), - }, - }, - Procedures: []*recipe.Procedure{ - { - Name: "test_procedure", - Type: "file", - Path: path.Join(defaulDirName, defaultValidFileName), - }, - }, - OutputTargets: []*recipe.OutputTarget{ - { - Name: "std", - Format: "yaml", - Type: "file", - Path: path.Join(defaulDirName), - }, - }, - } + l.Run("should return data and nil if no error is encountered", func() { loader := &core.Loader{} + pt := path.Join(defaultDirName, defaultDefinitionFileName) + _type := defaultValidType + format := defaultDefinitionFormat - actualValue, actualErr := loader.LoadFramework(rcp) + actualData, actualErr := loader.LoadData(pt, _type, format) - l.NotNil(actualValue) + l.NotNil(actualData) l.Nil(actualErr) }) } func (l *LoaderSuite) TearDownSuite() { - if err := os.RemoveAll(defaulDirName); err != nil { + if err := os.RemoveAll(defaultDirName); err != nil { panic(err) } } diff --git a/core/output_treatment.go b/core/output_treatment.go new file mode 100644 index 0000000..0028e13 --- /dev/null +++ b/core/output_treatment.go @@ -0,0 +1,51 @@ +package core + +import ( + "path" + + "github.com/gojek/optimus-extension-valor/model" + "github.com/gojek/optimus-extension-valor/registry/formatter" + "github.com/gojek/optimus-extension-valor/registry/io" +) + +func treatOutput(data *model.Data, output *model.Output) (bool, error) { + if output == nil { + return true, nil + } + outputError := &model.Error{} + for _, t := range output.Targets { + formatterFn, err := formatter.Formats.Get(jsonFormat, t.Format) + if err != nil { + outputError.Add(t.Name, err) + continue + } + writerFn, err := io.Writers.Get(t.Type) + if err != nil { + outputError.Add(t.Name, err) + continue + } + result, err := formatterFn(data.Content) + if err != nil { + outputError.Add(t.Name, err) + continue + } + writer := writerFn(output.TreatAs) + if err := writer.Write( + &model.Data{ + Type: data.Type, + Path: path.Join(t.Path, data.Path), + Content: result, + }, + ); err != nil { + outputError.Add(t.Name, err) + continue + } + } + if outputError.Length() > 0 { + return false, outputError + } + if len(output.Targets) > 0 && output.TreatAs == model.TreatmentError { + return false, nil + } + return true, nil +} diff --git a/core/validator.go b/core/validator.go index ab1fa7b..3a61d56 100644 --- a/core/validator.go +++ b/core/validator.go @@ -3,7 +3,6 @@ package core import ( "errors" "fmt" - "sync" "github.com/gojek/optimus-extension-valor/model" @@ -16,19 +15,18 @@ type Validator struct { } // NewValidator initializes Validator -func NewValidator(framework *model.Framework) (*Validator, model.Error) { - const defaultErrKey = "NewValidator" +func NewValidator(framework *model.Framework) (*Validator, error) { if framework == nil { - return nil, model.BuildError(defaultErrKey, errors.New("framework is nil")) + return nil, errors.New("framework is nil") } - outputError := make(model.Error) + outputError := &model.Error{} for i, sch := range framework.Schemas { if sch == nil { - key := fmt.Sprintf("%s [%d]", defaultErrKey, i) - outputError[key] = errors.New("schema is nil") + key := fmt.Sprintf("%d", i) + outputError.Add(key, errors.New("schema is nil")) } } - if len(outputError) > 0 { + if outputError.Length() > 0 { return nil, outputError } return &Validator{ @@ -37,57 +35,45 @@ func NewValidator(framework *model.Framework) (*Validator, model.Error) { } // Validate validates a Resource data against all schemas -func (v *Validator) Validate(resourceData *model.Data) model.Error { - const defaultErrKey = "Validate" +func (v *Validator) Validate(resourceData *model.Data) (bool, error) { if resourceData == nil { - return model.BuildError(defaultErrKey, errors.New("resource data is nil")) + return false, errors.New("resource data is nil") } - - wg := &sync.WaitGroup{} - mtx := &sync.Mutex{} - - outputError := make(model.Error) - for i, schema := range v.framework.Schemas { - wg.Add(1) - func(idx int, w *sync.WaitGroup, m *sync.Mutex, sch *model.Schema, rsc *model.Data) { - defer w.Done() - - if err := v.validateResourceToSchema(sch.Data, rsc); err != nil { - key := fmt.Sprintf("%s [%s: %d]", defaultErrKey, sch.Name, idx) - m.Lock() - outputError[key] = err - m.Unlock() + for _, schema := range v.framework.Schemas { + if schema.Data == nil { + return false, fmt.Errorf("schema data for [%s] is nil", schema.Name) + } + schemaLoader := gojsonschema.NewStringLoader(string(schema.Data.Content)) + recordLoader := gojsonschema.NewStringLoader(string(resourceData.Content)) + result, validateErr := gojsonschema.Validate(schemaLoader, recordLoader) + if validateErr != nil { + return false, validateErr + } + if result.Valid() { + continue + } + businessOutput := &model.Error{} + for _, r := range result.Errors() { + field := r.Field() + msg := r.Description() + businessOutput.Add(field, msg) + } + if businessOutput.Length() > 0 { + success, err := treatOutput( + &model.Data{ + Type: resourceData.Type, + Path: resourceData.Path, + Content: businessOutput.JSON(), + }, + schema.Output, + ) + if err != nil { + return false, err } - }(i, wg, mtx, schema, resourceData) - } - if len(outputError) > 0 { - return outputError - } - return nil -} - -func (v *Validator) validateResourceToSchema(schemaData *model.Data, resourceData *model.Data) model.Error { - const defaultErrKey = "validateResourceToSchema" - if schemaData == nil { - return model.BuildError(defaultErrKey, errors.New("schema data is nil")) - } - schemaLoader := gojsonschema.NewStringLoader(string(schemaData.Content)) - recordLoader := gojsonschema.NewStringLoader(string(resourceData.Content)) - result, validateErr := gojsonschema.Validate(schemaLoader, recordLoader) - if validateErr != nil { - return model.BuildError(defaultErrKey, validateErr) - } - if result.Valid() { - return nil - } - outputError := make(model.Error) - for _, r := range result.Errors() { - field := r.Field() - msg := r.Description() - outputError[field] = msg - } - if len(outputError) > 0 { - return outputError + if !success { + return false, nil + } + } } - return nil + return true, nil } diff --git a/core/validator_test.go b/core/validator_test.go index e2328ee..5b67eb1 100644 --- a/core/validator_test.go +++ b/core/validator_test.go @@ -15,17 +15,18 @@ type ValidatorSuite struct { } func (v *ValidatorSuite) TestValidate() { - v.Run("should return error if resource data is nil", func() { + v.Run("should return false and error if resource data is nil", func() { framework := &model.Framework{} var resourceData *model.Data = nil validator, _ := core.NewValidator(framework) - actualErr := validator.Validate(resourceData) + actualSuccess, actualErr := validator.Validate(resourceData) + v.False(actualSuccess) v.NotNil(actualErr) }) - v.Run("should return error if schema data is nil", func() { + v.Run("should return false and error if schema data is nil", func() { framework := &model.Framework{ Schemas: []*model.Schema{ { @@ -37,12 +38,13 @@ func (v *ValidatorSuite) TestValidate() { resourceData := &model.Data{} validator, _ := core.NewValidator(framework) - actualErr := validator.Validate(resourceData) + actualSuccess, actualErr := validator.Validate(resourceData) + v.False(actualSuccess) v.NotNil(actualErr) }) - v.Run("should return error if validation returns error", func() { + v.Run("should return false and nil if validation execution success but is business error", func() { schemaContent := `{ "title": "user_account", "description": "Schema to validate user_account.", @@ -80,8 +82,8 @@ func (v *ValidatorSuite) TestValidate() { } resourceContent := `{ "email": "valor@github.com", - "membership": "premium", - "is_active": 1 + "membership": "invalid", + "is_active": true } ` resourceData := &model.Data{ @@ -89,12 +91,13 @@ func (v *ValidatorSuite) TestValidate() { } validator, _ := core.NewValidator(framework) - actualErr := validator.Validate(resourceData) + actualSuccess, actualErr := validator.Validate(resourceData) - v.NotNil(actualErr) + v.True(actualSuccess) + v.Nil(actualErr) }) - v.Run("should return nil if validation success", func() { + v.Run("should return true and nil if validation execution and business are success", func() { schemaContent := `{ "title": "user_account", "description": "Schema to validate user_account.", @@ -141,8 +144,9 @@ func (v *ValidatorSuite) TestValidate() { } validator, _ := core.NewValidator(framework) - actualErr := validator.Validate(resourceData) + actualSuccess, actualErr := validator.Validate(resourceData) + v.True(actualSuccess) v.Nil(actualErr) }) } diff --git a/docs/command.md b/docs/command.md index 1f3e8a9..18fb094 100644 --- a/docs/command.md +++ b/docs/command.md @@ -78,7 +78,7 @@ Running the above command will execute all frameworks under `valor.yaml` recipe. Flag | Description | Format --- | --- | --- --batch-size | specify the number of data to be executed paralelly in one batch | it should be an integer more than 0 (zero). it is optional with default value 4 (four). ---progress-type | specify the progress type during execution | currently available: `verbose` (default) and `simple` +--progress-type | specify the progress type during execution | currently available: `progressive` (default) and `iterative` --recipe-path | customize the recipe that will be executed | it is optional. the value should be a valid recipe path This command also has sub-command. The currently available sub-commands are explained below. diff --git a/docs/recipe.md b/docs/recipe.md index 35b8c44..ce96ccf 100644 --- a/docs/recipe.md +++ b/docs/recipe.md @@ -13,7 +13,7 @@ Each recipe should contains one or more resources. An example: ```yaml resources: - name: user_account - type: dir + type: file path: ./example/resource format: json framework_names: @@ -42,22 +42,20 @@ Each resource is defined by a structure with the following fields: path the path where the resource should be read from - it has to follow the type format. for example, if the type is dir (to indicate local directory), then the format should follow how a directory path looks like + it has to follow the type format. for example, if the type is file (to indicate local file or directory), then the format should follow how a file path looks like ./example/resource type describes the type of path in order to get the resource - currently available: file and dir - dir + currently available: file + file @@ -91,10 +89,16 @@ frameworks: type: file format: json path: ./example/schema/user_account_rule.json + output: + treat_as: error + targets: + - name: std_output + type: std + format: yaml definitions: - name: memberships format: json - type: dir + type: file path: ./example/definition function: type: file @@ -104,11 +108,12 @@ frameworks: type: file format: jsonnet path: ./example/procedure/enrich_user_account.jsonnet - output_is_error: false - output_targets: - - name: std_output - type: std - format: json + output: + treat_as: success + targets: + - name: std_output + type: std + format: yaml ``` The following is the general constructs for a framework: @@ -119,7 +124,6 @@ name | true | defines the name of a particular framework. | it is suggested to b [schemas](#schema) | false | defines how to validate a resource. | it is an array of `schema` that will be executed _sequentially_ and _independently_.| for each schema, the output of validation is either a success or an error message. [definitions](#definition) | false | definitions are data input that might be required by **procedure**. **definitions** helps evaluation to be more efficient when external data is referenced multiple times. | it is an array of `definition` that defines how a definition should be prepared. | for each definition, the output is expected to be an array of JSON object. [procedures](#procedure) | false | defines how to evaluate a resource. | it is an array of `procedure` that will be executed sequentially with the ability to pass on information from one procedure to the next. | vary, dependig on how the procedure is constructed. -output_targets | false | defines how an output of validation and/or evaluation should be written out | it is an array of `output_target` that will be executed sequentially and independently. | vary, dependig on the validation and/or evaluation output ### Schema @@ -131,6 +135,13 @@ name: user_account_rule type: file format: json path: ./example/schema/user_account_rule.json +output: + treat_as: error + targets: + - name: std_output + type: std + format: yaml + path: ./out ... ``` @@ -139,7 +150,14 @@ Field | Description | Format name | the name of schema | it has to be unique within a framework only and should follow _`[a-z_]+`_ type | the type of data to be read from the path specified by **path** | currently available is `file` only format | the format being used to decode the data | currently available is `json` only, pointing that it's a JSON schema -path | the path where the schema rule to be read from | the valid format based on the **type** +path | the path where the schema rule to be read from | the valid format based on the **type**. if the specified path is a directory, then only the first file will be used as schema. +output | defines how output of the schema execution will be handled | it is optional. if it is being set, then its required fields should be specified. +output.treat_as | treatment that will be run against the output | currently availalbe: `info`, `warning`, `error`, `success`. if it is set to be `error`, then execution will not be continued. +output.targets | specifies the target output streams to write the result | it is an array of object, that needs to have a least one member +output.targets[].name | name of the output stream | it can be anything, but should be unique within the targets and should follow _`[a-z_]+`_ +output.targets[].type | the type of output stream | currently available: `file` and `std`, where the `std` is the standard output on console. +output.targets[].format | format output that will be written | currently available: `yaml` and `json` +output.targets[].path | the path where to write the output | it is required when the target type is `file` but not considered when it is set to be `std` _Note that every field mentioned above is mandatory unless stated otherwise._ @@ -179,22 +197,22 @@ Definition is external data that could be used by **procedures**. Definition is ... name: memberships format: json -type: dir +type: file path: ./example/definition function: - type: file - path: ./example/procedure/construct_membership_dictionary.jsonnet + type: file + path: ./example/procedure/construct_membership_dictionary.jsonnet ... ``` Field | Description | Format | Output --- | --- | --- | --- name | the name of definition | it has to be unique within a framework only and should follow _`[a-z_]+`_ | - -type | the type of data to be read from the path specified by **path** | currently available is `file` and `dir` | - +type | the type of data to be read from the path specified by **path** | currently available is `file` | - format | the format being used to decode the data | currently available is `json` and `yaml` | - path | the path where to read the actual data from | the valid format based on the **type** | - function | an optional instruction to build a definition, where the instruction follows the [Jsonnet](https://jsonnet.org/) format | - | dictionary where the key is the **name** and the value is up to the actual function defined under **function.path** -function.type | defines the type of path specified by **function.path** | it should be valid for the given **function.path** | - +function.type | defines the type of path specified by **function.path** | it should be valid for the given **function.path** with currently available is `file` | - function.path | defines the path where to read the actual function | should be valid according to the **function.type** | - _Note that every field mentioned above is mandatory unless stated otherwise._ @@ -274,7 +292,12 @@ name: enrich_user_account type: file format: jsonnet path: ./example/procedure/enrich_user_account.jsonnet -output_is_error: false +output: + treat_as: success + targets: + - name: std_output + type: std + format: yaml ... ``` @@ -284,7 +307,13 @@ name | the name of a procedure | it has to be unique within a framework only and type | the type of data to be read from the path specified by **path** | currently available is `file` only | - format | the format being used to decode the data | currently available is `jsonnet` only | - path | the path where to read the actual data from | the valid format based on the **type** | - -output_is_error | indicates whether output from a procedure is considered as error or not | it is optional, with the default value is `false`, where output is not considered as error +output | defines how output of the procedure execution will be handled | it is optional. if it is being set, then its required fields should be specified. +output.treat_as | treatment that will be run against the output | currently availalbe: `info`, `warning`, `error`, `success`. if it is set to be `error`, then execution will not be continued. +output.targets | specifies the target output streams to write the result | it is an array of object, that needs to have a least one member +output.targets[].name | name of the output stream | it can be anything, but should be unique within the targets and should follow _`[a-z_]+`_ +output.targets[].type | the type of output stream | currently available: `file` and `std`, where the `std` is the standard output on console. +output.targets[].format | format output that will be written | currently available: `yaml` and `json` +output.targets[].path | the path where to write the output | it is required when the target type is `file` but not considered when it is set to be `std` _Note that every field mentioned above is mandatory unless stated otherwise._ @@ -320,28 +349,8 @@ local membership_dict = definition['memberships']; ... ``` -this function wants to extract a definition named `memberships`, and use it as a reference to process its business flow. This function may or may not use the provided parameters, and may or may not return any output. It is entirely up to the user. In the above example, this function outputs an object. And since `output_is_error` is set to be `false`, then this value will be sent to the next pipeline, which can be: +this function wants to extract a definition named `memberships`, and use it as a reference to process its business flow. This function may or may not use the provided parameters, and may or may not return any output. It is entirely up to the user. In the above example, this function outputs an object. If the funcition returns output, then it will be sent to the next pipeline, which can be: * a new procedure, where this output will be sent as parameter under `previous`, or -* an output target, where this output will be written out to output stream, or +* an output, where this output will be written out to output stream, or * nothing, where the output will not be used. - -## Output Target - -Output target defines how an output would be written. One framework can zero or more output targets. If output target is not defined, then even if procedure returns output, it won't be written out. If output target is defined, but procedure doesn't return output, then no output will be written out. The basic construct of output target is like the following: - -```yaml -... -name: std_output -type: std -format: json -path: ./output -... -``` - -Field | Description | Format ---- | --- | --- -name | the name of output target | it has to be unique within a framework only and should follow _`[a-z_]+`_ -type | the type of output target | currently available: `std` (to write to standard output) and `dir` (to write to a directory) -format | the format for output | currently available: `json` and `yaml` -path | the path where to write the output | only being used when the **type** is set `dir` diff --git a/go.mod b/go.mod index 684d0bc..79f6243 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/gojek/optimus-extension-valor go 1.17 require ( + github.com/fatih/color v1.13.0 github.com/go-playground/validator/v10 v10.9.0 github.com/google/go-jsonnet v0.17.0 github.com/olekukonko/tablewriter v0.0.5 @@ -21,6 +22,8 @@ require ( github.com/go-playground/universal-translator v0.18.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect + github.com/mattn/go-colorable v0.1.9 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect diff --git a/go.sum b/go.sum index ee9889e..2ab5729 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -197,9 +199,14 @@ github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ic github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -407,6 +414,7 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -433,6 +441,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw= diff --git a/main.go b/main.go index 09538b9..d406d14 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "github.com/gojek/optimus-extension-valor/cmd" _ "github.com/gojek/optimus-extension-valor/plugin/endec" + _ "github.com/gojek/optimus-extension-valor/plugin/explorer" _ "github.com/gojek/optimus-extension-valor/plugin/formatter" _ "github.com/gojek/optimus-extension-valor/plugin/io" _ "github.com/gojek/optimus-extension-valor/plugin/progress" diff --git a/mocks/Reader.go b/mocks/Reader.go index 3aca1ef..5781e1f 100644 --- a/mocks/Reader.go +++ b/mocks/Reader.go @@ -12,33 +12,8 @@ type Reader struct { mock.Mock } -// ReadAll provides a mock function with given fields: -func (_m *Reader) ReadAll() ([]*model.Data, model.Error) { - ret := _m.Called() - - var r0 []*model.Data - if rf, ok := ret.Get(0).(func() []*model.Data); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*model.Data) - } - } - - var r1 model.Error - if rf, ok := ret.Get(1).(func() model.Error); ok { - r1 = rf() - } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(model.Error) - } - } - - return r0, r1 -} - -// ReadOne provides a mock function with given fields: -func (_m *Reader) ReadOne() (*model.Data, model.Error) { +// Read provides a mock function with given fields: +func (_m *Reader) Read() (*model.Data, error) { ret := _m.Called() var r0 *model.Data @@ -50,13 +25,11 @@ func (_m *Reader) ReadOne() (*model.Data, model.Error) { } } - var r1 model.Error - if rf, ok := ret.Get(1).(func() model.Error); ok { + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { r1 = rf() } else { - if ret.Get(1) != nil { - r1 = ret.Get(1).(model.Error) - } + r1 = ret.Error(1) } return r0, r1 diff --git a/mocks/Writer.go b/mocks/Writer.go index 5091c57..e7690a7 100644 --- a/mocks/Writer.go +++ b/mocks/Writer.go @@ -13,22 +13,14 @@ type Writer struct { } // Write provides a mock function with given fields: _a0 -func (_m *Writer) Write(_a0 ...*model.Data) model.Error { - _va := make([]interface{}, len(_a0)) - for _i := range _a0 { - _va[_i] = _a0[_i] - } - var _ca []interface{} - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) +func (_m *Writer) Write(_a0 *model.Data) error { + ret := _m.Called(_a0) - var r0 model.Error - if rf, ok := ret.Get(0).(func(...*model.Data) model.Error); ok { - r0 = rf(_a0...) + var r0 error + if rf, ok := ret.Get(0).(func(*model.Data) error); ok { + r0 = rf(_a0) } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(model.Error) - } + r0 = ret.Error(0) } return r0 diff --git a/model/core.go b/model/core.go index d4e4d12..5c63f93 100644 --- a/model/core.go +++ b/model/core.go @@ -13,6 +13,20 @@ var IsSkipResult = map[string]bool{ SkipNullValue: true, } +const ( + // TreatmentInfo is output treatment for info + TreatmentInfo OutputTreatment = "info" + // TreatmentWarning is output treatment for warning + TreatmentWarning OutputTreatment = "warning" + // TreatmentError is output treatment for error + TreatmentError OutputTreatment = "error" + // TreatmentSuccess is output treatment for success + TreatmentSuccess OutputTreatment = "success" +) + +// OutputTreatment is a type of treatment +type OutputTreatment string + // Data contains data information type Data struct { Type string @@ -29,11 +43,10 @@ type Resource struct { // Framework contains information on how to process a Resource type Framework struct { - Name string - Definitions []*Definition - Schemas []*Schema - Procedures []*Procedure - OutputTargets []*OutputTarget + Name string + Definitions []*Definition + Schemas []*Schema + Procedures []*Procedure } // Definition is the definition that could be used in a Procedure @@ -45,19 +58,26 @@ type Definition struct { // Schema contains information on Schema information defined by the user type Schema struct { - Name string - Data *Data + Name string + Data *Data + Output *Output } // Procedure contains information on Procedure information defined by the user type Procedure struct { - Name string - OutputIsError bool - Data *Data + Name string + Data *Data + Output *Output +} + +// Output describes how the last procedure output is written +type Output struct { + TreatAs OutputTreatment + Targets []*Target } -// OutputTarget describes the target of output for each Framework -type OutputTarget struct { +// Target defines how output is written to the targetted stream +type Target struct { Name string Format string Type string diff --git a/model/endec.go b/model/endec.go index df0c7e1..3b0337c 100644 --- a/model/endec.go +++ b/model/endec.go @@ -1,7 +1,7 @@ package model // Decode is a type to decode a raw data into a specified type output -type Decode func([]byte, interface{}) Error +type Decode func([]byte, interface{}) error // Encode is a type to encode a specifid type input into an output raw data -type Encode func(interface{}) ([]byte, Error) +type Encode func(interface{}) ([]byte, error) diff --git a/model/error.go b/model/error.go index f94bdb6..6330668 100644 --- a/model/error.go +++ b/model/error.go @@ -3,30 +3,48 @@ package model import ( "encoding/json" "fmt" + "sync" ) -// Error is error with in key-value formatted -type Error map[string]interface{} +// Error defines how execution error is constructed +type Error struct { + keyToValue map[string]interface{} -// Error returns error that represent the field error -func (e Error) Error() string { + initialized bool + mtx *sync.Mutex +} + +// Add adds a new error based on a specified key +func (e *Error) Add(key string, value interface{}) { + if !e.initialized { + e.keyToValue = make(map[string]interface{}) + e.mtx = &sync.Mutex{} + e.initialized = true + } + e.mtx.Lock() + e.keyToValue[key] = value + e.mtx.Unlock() +} + +// Error returns the summary of error +func (e *Error) Error() string { var output string - if len(e) > 0 { + if len(e.keyToValue) > 0 { var key string - for k := range e { + for k := range e.keyToValue { key = k break } output = fmt.Sprintf("error with key [%s]", key) - if len(e) > 1 { - output = output + " " + fmt.Sprintf("and %d others", len(e)-1) + if len(e.keyToValue) > 1 { + output += fmt.Sprintf(" and %d others", len(e.keyToValue)-1) } } return output } -// JSON converts field error into its JSON representation -func (e Error) JSON() []byte { +// JSON returns the complete error message representation +func (e *Error) JSON() []byte { mapError := e.buildMap() output, err := json.MarshalIndent(mapError, "", " ") if err != nil { @@ -35,9 +53,14 @@ func (e Error) JSON() []byte { return output } -func (e Error) buildMap() map[string]interface{} { +// Length returns the number of errors stored so far +func (e *Error) Length() int { + return len(e.keyToValue) +} + +func (e *Error) buildMap() map[string]interface{} { output := make(map[string]interface{}) - for key, value := range e { + for key, value := range e.keyToValue { if customErr, ok := value.(Error); ok { mV := customErr.buildMap() output[key] = mV @@ -51,10 +74,3 @@ func (e Error) buildMap() map[string]interface{} { } return output } - -// BuildError builds a new error based on key and value -func BuildError(key string, value interface{}) Error { - return map[string]interface{}{ - key: value, - } -} diff --git a/model/explorer.go b/model/explorer.go new file mode 100644 index 0000000..a8af0f0 --- /dev/null +++ b/model/explorer.go @@ -0,0 +1,4 @@ +package model + +// ExplorePath explores path from its root with filter +type ExplorePath func(root string, filter func(string) bool) ([]string, error) diff --git a/model/format.go b/model/format.go index cf13068..96b70a9 100644 --- a/model/format.go +++ b/model/format.go @@ -1,4 +1,4 @@ package model // Format is a type to format input from one input to another -type Format func([]byte) ([]byte, Error) +type Format func([]byte) ([]byte, error) diff --git a/model/io.go b/model/io.go index 6bdb5b8..e2bc905 100644 --- a/model/io.go +++ b/model/io.go @@ -1,21 +1,17 @@ package model -// Reader is a contract to read from a single source +// Reader is a contract to read from a source type Reader interface { - ReadOne() (*Data, Error) - ReadAll() ([]*Data, Error) + Read() (*Data, error) } // Writer is a contract to write data type Writer interface { - Write(...*Data) Error + Write(*Data) error } // GetPath gets the path type GetPath func() string -// FilterPath filters the path with True value will be processed -type FilterPath func(string) bool - // PostProcess post processes data -type PostProcess func(path string, content []byte) (*Data, Error) +type PostProcess func(path string, content []byte) (*Data, error) diff --git a/model/progress.go b/model/progress.go index b3e2454..1814c08 100644 --- a/model/progress.go +++ b/model/progress.go @@ -2,7 +2,7 @@ package model // Progress is a contract for process progress type Progress interface { - Increment() + Increase(int) Wait() } diff --git a/plugin/endec/json/json.go b/plugin/endec/json/json.go index 2edc690..5d7ed5d 100644 --- a/plugin/endec/json/json.go +++ b/plugin/endec/json/json.go @@ -11,11 +11,10 @@ const format = "json" // NewEncode initializes JSON encoding function func NewEncode() model.Encode { - const defaultErrKey = "NewEncode" - return func(i interface{}) ([]byte, model.Error) { + return func(i interface{}) ([]byte, error) { output, err := json.Marshal(i) if err != nil { - return nil, model.BuildError(defaultErrKey, err) + return nil, err } return output, nil } @@ -23,10 +22,9 @@ func NewEncode() model.Encode { // NewDecode initializes JSON decodign function func NewDecode() model.Decode { - const defaultErrKey = "NewDecode" - return func(b []byte, i interface{}) model.Error { + return func(b []byte, i interface{}) error { if err := json.Unmarshal(b, i); err != nil { - return model.BuildError(defaultErrKey, err) + return err } return nil } diff --git a/plugin/endec/json/json_test.go b/plugin/endec/json/json_test.go index c7e82e7..6160713 100644 --- a/plugin/endec/json/json_test.go +++ b/plugin/endec/json/json_test.go @@ -9,8 +9,6 @@ import ( ) func TestNewEncode(t *testing.T) { - const defaultErrKey = "NewEncode" - t.Run("should return nil and error if error when executing", func(t *testing.T) { message := func() {} encode := json.NewEncode() @@ -33,8 +31,6 @@ func TestNewEncode(t *testing.T) { } func TestNewDecode(t *testing.T) { - const defaultErrKey = "NewDecode" - t.Run("should return error if error when executing", func(t *testing.T) { input := []byte("message") var output int diff --git a/plugin/endec/yaml/yaml.go b/plugin/endec/yaml/yaml.go index 8bd1bf4..78d477e 100644 --- a/plugin/endec/yaml/yaml.go +++ b/plugin/endec/yaml/yaml.go @@ -13,8 +13,7 @@ const format = "yaml" // NewEncode initializes YAML encoder function func NewEncode() model.Encode { - const defaultErrKey = "NewEncode" - return func(i interface{}) ([]byte, model.Error) { + return func(i interface{}) ([]byte, error) { var recoverErr error output, err := func() ([]byte, error) { defer func() { @@ -25,10 +24,10 @@ func NewEncode() model.Encode { return yaml.Marshal(i) }() if err != nil { - return nil, model.BuildError(defaultErrKey, err) + return nil, err } if recoverErr != nil { - return nil, model.BuildError(defaultErrKey, recoverErr) + return nil, recoverErr } return output, nil } @@ -36,10 +35,9 @@ func NewEncode() model.Encode { // NewDecode initializes YAML decoder function func NewDecode() model.Decode { - const defaultErrKey = "NewDecode" - return func(b []byte, i interface{}) model.Error { + return func(b []byte, i interface{}) error { if err := yaml.Unmarshal(b, i); err != nil { - return model.BuildError(defaultErrKey, err) + return err } return nil } diff --git a/plugin/endec/yaml/yaml_test.go b/plugin/endec/yaml/yaml_test.go index 73adc0f..46e02f3 100644 --- a/plugin/endec/yaml/yaml_test.go +++ b/plugin/endec/yaml/yaml_test.go @@ -4,12 +4,11 @@ import ( "testing" "github.com/gojek/optimus-extension-valor/plugin/endec/yaml" + "github.com/stretchr/testify/assert" ) func TestNewEncode(t *testing.T) { - const defaultErrKey = "NewEncode" - t.Run("should return nil and error if error when executing", func(t *testing.T) { message := func() {} encode := yaml.NewEncode() @@ -32,8 +31,6 @@ func TestNewEncode(t *testing.T) { } func TestNewDecode(t *testing.T) { - const defaultErrKey = "NewDecode" - t.Run("should return error if error when executing", func(t *testing.T) { input := []byte("message") var output int diff --git a/plugin/explorer/file.go b/plugin/explorer/file.go new file mode 100644 index 0000000..15c107b --- /dev/null +++ b/plugin/explorer/file.go @@ -0,0 +1,35 @@ +package explorer + +import ( + "io/fs" + "path/filepath" + + "github.com/gojek/optimus-extension-valor/registry/explorer" +) + +const fileType = "file" + +// ExploreFilePath explores file path from its root based on its filter +func ExploreFilePath(root string, filter func(string) bool) ([]string, error) { + var output []string + if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + if filter == nil || filter(path) { + output = append(output, path) + } + } + return nil + }); err != nil { + return nil, err + } + return output, nil +} + +func init() { + if err := explorer.Explorers.Register(fileType, ExploreFilePath); err != nil { + panic(err) + } +} diff --git a/plugin/formatter/json/json.go b/plugin/formatter/json/json.go index 5a003b8..238eb87 100644 --- a/plugin/formatter/json/json.go +++ b/plugin/formatter/json/json.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" - "github.com/gojek/optimus-extension-valor/model" "github.com/gojek/optimus-extension-valor/registry/formatter" "gopkg.in/yaml.v3" @@ -16,7 +15,7 @@ const ( ) // ToJSON formats input from JSON to JSON, which does nothing -func ToJSON(input []byte) ([]byte, model.Error) { +func ToJSON(input []byte) ([]byte, error) { output := make([]byte, len(input)) for i := 0; i < len(input); i++ { output[i] = input[i] @@ -25,18 +24,17 @@ func ToJSON(input []byte) ([]byte, model.Error) { } // ToYAML formats input from JSON to YAML -func ToYAML(input []byte) ([]byte, model.Error) { - const defaultErrKey = "ToYAML" +func ToYAML(input []byte) ([]byte, error) { var t interface{} err := json.Unmarshal(input, &t) if err != nil { - return nil, model.BuildError(defaultErrKey, err) + return nil, err } var b bytes.Buffer y := yaml.NewEncoder(&b) y.SetIndent(2) if err := y.Encode(t); err != nil { - return nil, model.BuildError(defaultErrKey, err) + return nil, err } return b.Bytes(), nil } diff --git a/plugin/formatter/yaml/yaml.go b/plugin/formatter/yaml/yaml.go index 46ae0c4..040e8ef 100644 --- a/plugin/formatter/yaml/yaml.go +++ b/plugin/formatter/yaml/yaml.go @@ -3,29 +3,32 @@ package yaml import ( "encoding/json" - "github.com/gojek/optimus-extension-valor/model" "github.com/gojek/optimus-extension-valor/registry/formatter" "gopkg.in/yaml.v3" ) +const ( + jsonType = "json" + yamlType = "yaml" +) + // ToJSON formats input from YAML to JSON -func ToJSON(input []byte) ([]byte, model.Error) { - const defaultErrKey = "ToJSON" +func ToJSON(input []byte) ([]byte, error) { var t interface{} err := yaml.Unmarshal(input, &t) if err != nil { - return nil, model.BuildError(defaultErrKey, err) + return nil, err } output, err := json.MarshalIndent(t, "", " ") if err != nil { - return nil, model.BuildError(defaultErrKey, err) + return nil, err } return output, nil } func init() { - err := formatter.Formats.Register("yaml", "json", ToJSON) + err := formatter.Formats.Register(yamlType, jsonType, ToJSON) if err != nil { panic(err) } diff --git a/plugin/io/dir/dir.go b/plugin/io/dir/dir.go deleted file mode 100644 index 08e64d6..0000000 --- a/plugin/io/dir/dir.go +++ /dev/null @@ -1,200 +0,0 @@ -package dir - -import ( - "errors" - "fmt" - "io/fs" - "io/ioutil" - "os" - "path" - "path/filepath" - "sync" - - "github.com/gojek/optimus-extension-valor/model" - "github.com/gojek/optimus-extension-valor/registry/io" -) - -const _type = "dir" - -// Dir represents directory operation -type Dir struct { - getPath model.GetPath - filterPath model.FilterPath - postProcess model.PostProcess -} - -func (d *Dir) Write(dataList ...*model.Data) model.Error { - const defaultErrKey = "Write" - if len(dataList) == 0 { - return model.BuildError(defaultErrKey, errors.New("data is empty")) - } - - wg := &sync.WaitGroup{} - mtx := &sync.Mutex{} - - outputError := make(model.Error) - for _, data := range dataList { - wg.Add(1) - - go func(w *sync.WaitGroup, m *sync.Mutex, dt *model.Data) { - defer w.Done() - - key := fmt.Sprintf("%s [%s]", defaultErrKey, dt.Path) - dirPath, _ := path.Split(dt.Path) - if err := os.MkdirAll(dirPath, os.ModePerm); err != nil { - m.Lock() - outputError[key] = err - m.Unlock() - return - } - if err := ioutil.WriteFile(dt.Path, dt.Content, os.ModePerm); err != nil { - m.Lock() - outputError[key] = err - m.Unlock() - return - } - }(wg, mtx, data) - } - wg.Wait() - if len(outputError) > 0 { - return outputError - } - return nil -} - -// ReadAll reads all files in a directory -func (d *Dir) ReadAll() ([]*model.Data, model.Error) { - const defaultErrKey = "ReadAll" - if err := d.validate(); err != nil { - return nil, err - } - dirPath := d.getPath() - filePaths, err := d.getFilePaths(dirPath, d.filterPath) - if err != nil { - return nil, err - } - - wg := &sync.WaitGroup{} - mtx := &sync.Mutex{} - - outputData := make([]*model.Data, len(filePaths)) - outputError := make(model.Error) - for i, path := range filePaths { - wg.Add(1) - - go func(idx int, w *sync.WaitGroup, m *sync.Mutex, pt string) { - defer w.Done() - - key := fmt.Sprintf("%s [%s]", defaultErrKey, pt) - content, err := ioutil.ReadFile(pt) - if err != nil { - m.Lock() - outputError[key] = err - m.Unlock() - } else { - data, err := d.postProcess(pt, content) - if err != nil { - m.Lock() - outputError[key] = err - m.Unlock() - } else { - m.Lock() - outputData[idx] = data - m.Unlock() - } - } - }(i, wg, mtx, path) - } - wg.Wait() - if len(outputError) > 0 { - return nil, outputError - } - return outputData, nil -} - -// ReadOne reads the first file in a directory -func (d *Dir) ReadOne() (*model.Data, model.Error) { - const defaultErrKey = "ReadOne" - if err := d.validate(); err != nil { - return nil, err - } - dirPath := d.getPath() - filePaths, pathErr := d.getFilePaths(dirPath, d.filterPath) - if pathErr != nil { - return nil, pathErr - } - content, readErr := ioutil.ReadFile(filePaths[0]) - if readErr != nil { - return nil, model.BuildError(defaultErrKey, readErr) - } - return d.postProcess(filePaths[0], content) -} - -func (d *Dir) getFilePaths(dirPath string, filterPath model.FilterPath) ([]string, model.Error) { - const defaultErrKey = "getFilePaths" - var output []string - err := filepath.Walk(dirPath, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - if filterPath == nil || filterPath(path) { - output = append(output, path) - } - } - return nil - }) - if err != nil { - return nil, model.BuildError(defaultErrKey, err) - } - if len(output) == 0 { - return nil, model.BuildError(defaultErrKey, errors.New("no file path is found based on filter")) - } - return output, nil -} - -func (d *Dir) validate() model.Error { - const defaultErrKey = "validate" - if d.getPath == nil { - return model.BuildError(defaultErrKey, errors.New("getPath is nil")) - } - if d.postProcess == nil { - return model.BuildError(defaultErrKey, errors.New("postProcess is nil")) - } - return nil -} - -// NewReader initializes dir Reader -func NewReader( - getPath model.GetPath, - filterPath model.FilterPath, - postProcess model.PostProcess, -) *Dir { - return &Dir{ - getPath: getPath, - filterPath: filterPath, - postProcess: postProcess, - } -} - -// NewWriter initializes dir Writer -func NewWriter() *Dir { - return &Dir{} -} - -func init() { - err := io.Readers.Register(_type, func( - getPath model.GetPath, - filterPath model.FilterPath, - postProcess model.PostProcess, - ) model.Reader { - return NewReader(getPath, filterPath, postProcess) - }) - if err != nil { - panic(err) - } - err = io.Writers.Register(_type, NewWriter()) - if err != nil { - panic(err) - } -} diff --git a/plugin/io/dir/dir_test.go b/plugin/io/dir/dir_test.go deleted file mode 100644 index 171184d..0000000 --- a/plugin/io/dir/dir_test.go +++ /dev/null @@ -1,281 +0,0 @@ -package dir_test - -import ( - "errors" - "io/ioutil" - "os" - "path" - "testing" - - "github.com/gojek/optimus-extension-valor/model" - "github.com/gojek/optimus-extension-valor/plugin/io/dir" - - "github.com/stretchr/testify/suite" -) - -const ( - defaulDirName = "./out" - defaultFileName = "test.yaml" - defaultContent = "message" -) - -type DirSuite struct { - suite.Suite -} - -func (d *DirSuite) SetupSuite() { - if err := os.MkdirAll(defaulDirName, os.ModePerm); err != nil { - panic(err) - } - filePath := path.Join(defaulDirName, defaultFileName) - if err := ioutil.WriteFile(filePath, []byte(defaultContent), os.ModePerm); err != nil { - panic(err) - } -} - -func (d *DirSuite) TestWrite() { - d.Run("should return error if data list is empty", func() { - var dataList []*model.Data - writer := dir.NewWriter() - - actualErr := writer.Write(dataList...) - - d.NotNil(actualErr) - }) - - d.Run("should return error if there's error during write", func() { - dataList := []*model.Data{ - { - Content: []byte(defaultContent), - Path: path.Join(defaulDirName, defaultFileName, "test"), - }, - { - Content: []byte(defaultContent), - Path: defaulDirName, - }, - } - writer := dir.NewWriter() - - actualErr := writer.Write(dataList...) - - d.NotNil(actualErr) - }) - - d.Run("should return error if there's error during write", func() { - dataList := []*model.Data{ - { - Content: []byte(defaultContent), - Path: path.Join(defaulDirName, defaultFileName), - }, - } - writer := dir.NewWriter() - - actualErr := writer.Write(dataList...) - - d.Nil(actualErr) - }) -} - -func (d *DirSuite) TestReadAll() { - d.Run("should return error if getPath nil", func() { - var getPath model.GetPath = nil - var filterPath model.FilterPath = func(s string) bool { - return true - } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return &model.Data{ - Content: content, - Path: path, - }, nil - } - reader := dir.NewReader(getPath, filterPath, postProcess) - - actualData, actualErr := reader.ReadAll() - - d.Nil(actualData) - d.NotNil(actualErr) - }) - - d.Run("should return error if postProcess nil", func() { - var getPath model.GetPath = func() string { - return defaulDirName - } - var filterPath model.FilterPath = func(s string) bool { - return true - } - var postProcess model.PostProcess = nil - reader := dir.NewReader(getPath, filterPath, postProcess) - - actualData, actualErr := reader.ReadAll() - - d.Nil(actualData) - d.NotNil(actualErr) - }) - - d.Run("should return error if no file paths found nil", func() { - var getPath model.GetPath = func() string { - return defaulDirName - } - var filterPath model.FilterPath = func(s string) bool { - return false - } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return &model.Data{ - Content: content, - Path: path, - }, nil - } - reader := dir.NewReader(getPath, filterPath, postProcess) - - actualData, actualErr := reader.ReadAll() - - d.Nil(actualData) - d.NotNil(actualErr) - }) - - d.Run("should return error if error is found when post process", func() { - var getPath model.GetPath = func() string { - return defaulDirName - } - var filterPath model.FilterPath = func(s string) bool { - return true - } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return nil, model.BuildError("test", errors.New("test error")) - } - reader := dir.NewReader(getPath, filterPath, postProcess) - - actualData, actualErr := reader.ReadAll() - - d.Nil(actualData) - d.NotNil(actualErr) - }) - - d.Run("should return value if no error is found", func() { - var getPath model.GetPath = func() string { - return defaulDirName - } - var filterPath model.FilterPath = func(s string) bool { - return true - } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return &model.Data{ - Content: content, - Path: path, - }, nil - } - reader := dir.NewReader(getPath, filterPath, postProcess) - - actualData, actualErr := reader.ReadAll() - - d.NotNil(actualData) - d.Nil(actualErr) - }) -} - -func (d *DirSuite) TestReadOne() { - d.Run("should return error if getPath nil", func() { - var getPath model.GetPath = nil - var filterPath model.FilterPath = func(s string) bool { - return true - } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return &model.Data{ - Content: content, - Path: path, - }, nil - } - reader := dir.NewReader(getPath, filterPath, postProcess) - - actualData, actualErr := reader.ReadOne() - - d.Nil(actualData) - d.NotNil(actualErr) - }) - - d.Run("should return error if postProcess nil", func() { - var getPath model.GetPath = func() string { - return defaulDirName - } - var filterPath model.FilterPath = func(s string) bool { - return true - } - var postProcess model.PostProcess = nil - reader := dir.NewReader(getPath, filterPath, postProcess) - - actualData, actualErr := reader.ReadOne() - - d.Nil(actualData) - d.NotNil(actualErr) - }) - - d.Run("should return error if no file paths found nil", func() { - var getPath model.GetPath = func() string { - return defaulDirName - } - var filterPath model.FilterPath = func(s string) bool { - return false - } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return &model.Data{ - Content: content, - Path: path, - }, nil - } - reader := dir.NewReader(getPath, filterPath, postProcess) - - actualData, actualErr := reader.ReadOne() - - d.Nil(actualData) - d.NotNil(actualErr) - }) - - d.Run("should return error if error is found when post process", func() { - var getPath model.GetPath = func() string { - return defaulDirName - } - var filterPath model.FilterPath = func(s string) bool { - return true - } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return nil, model.BuildError("test", errors.New("test error")) - } - reader := dir.NewReader(getPath, filterPath, postProcess) - - actualData, actualErr := reader.ReadOne() - - d.Nil(actualData) - d.NotNil(actualErr) - }) - - d.Run("should return value if no error is found", func() { - var getPath model.GetPath = func() string { - return defaulDirName - } - var filterPath model.FilterPath = func(s string) bool { - return true - } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return &model.Data{ - Content: content, - Path: path, - }, nil - } - reader := dir.NewReader(getPath, filterPath, postProcess) - - actualData, actualErr := reader.ReadOne() - - d.NotNil(actualData) - d.Nil(actualErr) - }) -} - -func (d *DirSuite) TearDownSuite() { - if err := os.RemoveAll(defaulDirName); err != nil { - panic(err) - } -} - -func TestDirSuite(t *testing.T) { - suite.Run(t, &DirSuite{}) -} diff --git a/plugin/io/file/file.go b/plugin/io/file/file.go index 405f0db..c6a3748 100644 --- a/plugin/io/file/file.go +++ b/plugin/io/file/file.go @@ -3,6 +3,8 @@ package file import ( "errors" "io/ioutil" + "os" + "path" "github.com/gojek/optimus-extension-valor/model" "github.com/gojek/optimus-extension-valor/registry/io" @@ -16,42 +18,39 @@ type File struct { postProcess model.PostProcess } -// ReadOne reads one file from a path -func (f *File) ReadOne() (*model.Data, model.Error) { - const defaultErrKey = "ReadOne" +// Read reads file from a path +func (f *File) Read() (*model.Data, error) { if err := f.validate(); err != nil { return nil, err } path := f.getPath() content, err := ioutil.ReadFile(path) if err != nil { - return nil, model.BuildError(defaultErrKey, err) + return nil, err } return f.postProcess(path, content) } -// ReadAll reads one file from a path but is returned as a slice -func (f *File) ReadAll() ([]*model.Data, model.Error) { - const defaultErrKey = "ReadAll" - if err := f.validate(); err != nil { - return nil, err +func (f *File) validate() error { + if f.getPath == nil { + return errors.New("getPath is nil") } - data, err := f.ReadOne() - if err != nil { - return nil, err + if f.postProcess == nil { + return errors.New("postProcess is nil") } - return []*model.Data{data}, nil + return nil } -func (f *File) validate() model.Error { - const defaultErrKey = "validate" - if f.getPath == nil { - return model.BuildError(defaultErrKey, errors.New("getPath is nil")) +// Write writes data to destination +func (f *File) Write(data *model.Data) error { + if data == nil { + return errors.New("data is nil") } - if f.postProcess == nil { - return model.BuildError(defaultErrKey, errors.New("postProcess is nil")) + dirPath, _ := path.Split(data.Path) + if err := os.MkdirAll(dirPath, os.ModePerm); err != nil { + return err } - return nil + return ioutil.WriteFile(data.Path, data.Content, os.ModePerm) } // New initializes File based on path @@ -63,14 +62,18 @@ func New(getPath model.GetPath, postProcess model.PostProcess) *File { } func init() { - err := io.Readers.Register(_type, func( - getPath model.GetPath, - filterPath model.FilterPath, - postProcess model.PostProcess, - ) model.Reader { - return New(getPath, postProcess) - }) - if err != nil { + if err := io.Readers.Register(_type, + func(getPath model.GetPath, postProcess model.PostProcess) model.Reader { + return New(getPath, postProcess) + }, + ); err != nil { + panic(err) + } + if err := io.Writers.Register(_type, + func(treatment model.OutputTreatment) model.Writer { + return New(nil, nil) + }, + ); err != nil { panic(err) } } diff --git a/plugin/io/file/file_test.go b/plugin/io/file/file_test.go index 3d65987..5318283 100644 --- a/plugin/io/file/file_test.go +++ b/plugin/io/file/file_test.go @@ -14,7 +14,7 @@ import ( ) const ( - defaulDirName = "./out" + defaultDirName = "./out" defaultFileName = "test.yaml" defaultContent = "message" ) @@ -24,19 +24,19 @@ type FileSuite struct { } func (f *FileSuite) SetupSuite() { - if err := os.MkdirAll(defaulDirName, os.ModePerm); err != nil { + if err := os.MkdirAll(defaultDirName, os.ModePerm); err != nil { panic(err) } - filePath := path.Join(defaulDirName, defaultFileName) + filePath := path.Join(defaultDirName, defaultFileName) if err := ioutil.WriteFile(filePath, []byte(defaultContent), os.ModePerm); err != nil { panic(err) } } -func (f *FileSuite) TestReadAll() { +func (f *FileSuite) TestRead() { f.Run("should return error if getPath nil", func() { var getPath model.GetPath = nil - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { + var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, error) { return &model.Data{ Content: content, Path: path, @@ -44,7 +44,7 @@ func (f *FileSuite) TestReadAll() { } reader := file.New(getPath, postProcess) - actualData, actualErr := reader.ReadAll() + actualData, actualErr := reader.Read() f.Nil(actualData) f.NotNil(actualErr) @@ -52,12 +52,12 @@ func (f *FileSuite) TestReadAll() { f.Run("should return error if postProcess nil", func() { var getPath model.GetPath = func() string { - return defaulDirName + return defaultDirName } var postProcess model.PostProcess = nil reader := file.New(getPath, postProcess) - actualData, actualErr := reader.ReadAll() + actualData, actualErr := reader.Read() f.Nil(actualData) f.NotNil(actualErr) @@ -65,14 +65,14 @@ func (f *FileSuite) TestReadAll() { f.Run("should return error if error is found when post process", func() { var getPath model.GetPath = func() string { - return defaulDirName + return defaultDirName } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return nil, model.BuildError("test", errors.New("test error")) + var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, error) { + return nil, errors.New("test error") } reader := file.New(getPath, postProcess) - actualData, actualErr := reader.ReadAll() + actualData, actualErr := reader.Read() f.Nil(actualData) f.NotNil(actualErr) @@ -80,9 +80,9 @@ func (f *FileSuite) TestReadAll() { f.Run("should return value if no error is found", func() { var getPath model.GetPath = func() string { - return path.Join(defaulDirName, defaultFileName) + return path.Join(defaultDirName, defaultFileName) } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { + var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, error) { return &model.Data{ Content: content, Path: path, @@ -90,79 +90,40 @@ func (f *FileSuite) TestReadAll() { } reader := file.New(getPath, postProcess) - actualData, actualErr := reader.ReadAll() + actualData, actualErr := reader.Read() f.NotNil(actualData) f.Nil(actualErr) }) } -func (f *FileSuite) TestReadOne() { - f.Run("should return error if getPath nil", func() { - var getPath model.GetPath = nil - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return &model.Data{ - Content: content, - Path: path, - }, nil - } - reader := file.New(getPath, postProcess) - - actualData, actualErr := reader.ReadOne() - - f.Nil(actualData) - f.NotNil(actualErr) - }) - - f.Run("should return error if postProcess nil", func() { - var getPath model.GetPath = func() string { - return defaulDirName - } - var postProcess model.PostProcess = nil - reader := file.New(getPath, postProcess) - - actualData, actualErr := reader.ReadOne() +func (f *FileSuite) TestWrite() { + f.Run("should return error if data is nil", func() { + reader := file.New(nil, nil) + var data *model.Data = nil - f.Nil(actualData) - f.NotNil(actualErr) - }) - - f.Run("should return error if error is found when post process", func() { - var getPath model.GetPath = func() string { - return defaulDirName - } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return nil, model.BuildError("test", errors.New("test error")) - } - reader := file.New(getPath, postProcess) - - actualData, actualErr := reader.ReadOne() + actualErr := reader.Write(data) - f.Nil(actualData) f.NotNil(actualErr) }) - f.Run("should return value if no error is found", func() { - var getPath model.GetPath = func() string { - return path.Join(defaulDirName, defaultFileName) - } - var postProcess model.PostProcess = func(path string, content []byte) (*model.Data, model.Error) { - return &model.Data{ - Content: content, - Path: path, - }, nil + f.Run("should return result of write", func() { + defer func() { os.RemoveAll("./output") }() + data := &model.Data{ + Type: "file", + Path: path.Join(defaultDirName, defaultFileName), + Content: []byte(defaultContent), } - reader := file.New(getPath, postProcess) + reader := file.New(nil, nil) - actualData, actualErr := reader.ReadOne() + actualErr := reader.Write(data) - f.NotNil(actualData) f.Nil(actualErr) }) } func (f *FileSuite) TearDownSuite() { - if err := os.RemoveAll(defaulDirName); err != nil { + if err := os.RemoveAll(defaultDirName); err != nil { panic(err) } } diff --git a/plugin/io/registry.go b/plugin/io/registry.go index 57222ed..481eb87 100644 --- a/plugin/io/registry.go +++ b/plugin/io/registry.go @@ -1,7 +1,6 @@ package io import ( - _ "github.com/gojek/optimus-extension-valor/plugin/io/dir" // init Dir io _ "github.com/gojek/optimus-extension-valor/plugin/io/file" // init File io _ "github.com/gojek/optimus-extension-valor/plugin/io/std" // init Std io ) diff --git a/plugin/io/std/std.go b/plugin/io/std/std.go index 6c18e94..8b1477d 100644 --- a/plugin/io/std/std.go +++ b/plugin/io/std/std.go @@ -1,38 +1,61 @@ package std import ( - "fmt" + "errors" "os" + "strings" "github.com/gojek/optimus-extension-valor/model" "github.com/gojek/optimus-extension-valor/registry/io" + + "github.com/fatih/color" ) const _type = "std" // Std is an stdout writer type Std struct { + treatment model.OutputTreatment } -func (s *Std) Write(dataList ...*model.Data) model.Error { - const defaultErrKey = "Write" - for _, d := range dataList { - output := fmt.Sprintf("%s\n%s\n", d.Path, string(d.Content)) - _, err := os.Stdout.WriteString(output) - if err != nil { - return model.BuildError(defaultErrKey, err) - } +func (s *Std) Write(data *model.Data) error { + if data == nil { + return errors.New("data is nil") + } + color.NoColor = false + var colorize *color.Color + switch s.treatment { + case model.TreatmentError: + colorize = color.New(color.FgHiRed) + case model.TreatmentWarning: + colorize = color.New(color.FgHiYellow) + case model.TreatmentSuccess: + colorize = color.New(color.FgHiGreen) + default: + colorize = color.New(color.FgHiWhite) } - return nil + separator := strings.Repeat("-", len(data.Path)) + output := colorize.Sprintf("%s\n%s\n%s\n%s\n", + separator, + data.Path, + separator, + string(data.Content), + ) + _, err := os.Stdout.WriteString(output) + return err } // New initializes standard input and output -func New() *Std { - return &Std{} +func New(treatment model.OutputTreatment) *Std { + return &Std{ + treatment: treatment, + } } func init() { - err := io.Writers.Register(_type, New()) + err := io.Writers.Register(_type, func(treatment model.OutputTreatment) model.Writer { + return New(treatment) + }) if err != nil { panic(err) } diff --git a/plugin/io/std/std_test.go b/plugin/io/std/std_test.go new file mode 100644 index 0000000..f1bf695 --- /dev/null +++ b/plugin/io/std/std_test.go @@ -0,0 +1,42 @@ +package std_test + +import ( + "testing" + + "github.com/gojek/optimus-extension-valor/model" + "github.com/gojek/optimus-extension-valor/plugin/io/std" + + "github.com/stretchr/testify/suite" +) + +type StdSuite struct { + suite.Suite +} + +func (f *StdSuite) TestWrite() { + f.Run("should return error if data is nil", func() { + reader := std.New(model.TreatmentInfo) + var data *model.Data = nil + + actualErr := reader.Write(data) + + f.NotNil(actualErr) + }) + + f.Run("should return result of write", func() { + data := &model.Data{ + Type: "std", + Path: "./output/test.file", + Content: []byte("test content"), + } + reader := std.New(model.TreatmentInfo) + + actualErr := reader.Write(data) + + f.Nil(actualErr) + }) +} + +func TestFileSuite(t *testing.T) { + suite.Run(t, &StdSuite{}) +} diff --git a/plugin/progress/simple.go b/plugin/progress/iterative.go similarity index 62% rename from plugin/progress/simple.go rename to plugin/progress/iterative.go index 4b613d4..53ace6e 100644 --- a/plugin/progress/simple.go +++ b/plugin/progress/iterative.go @@ -9,10 +9,10 @@ import ( "github.com/gojek/optimus-extension-valor/registry/progress" ) -const simpleType = "simple" +const iterativeType = "iterative" -// Simple defines how a simple progress bar should be executed -type Simple struct { +// Iterative defines how a iterative progress bar should be executed +type Iterative struct { name string total int @@ -23,12 +23,16 @@ type Simple struct { startTime time.Time } -// Increment increments the progress -func (s *Simple) Increment() { +// Increase increases the progress by the number +func (s *Iterative) Increase(num int) { if s.currentCounter >= s.total || s.finished { return } - s.currentCounter++ + increment := num + if s.currentCounter+increment > s.total { + increment = s.total - s.currentCounter + } + s.currentCounter += increment currentPercentage := 100 * s.currentCounter / s.total if currentPercentage > s.previousPercentage { @@ -45,16 +49,16 @@ func (s *Simple) Increment() { } // Wait finishes the progress -func (s *Simple) Wait() { +func (s *Iterative) Wait() { if !s.finished { fmt.Printf("total elapsed: %v\n", time.Now().Sub(s.startTime)) } s.finished = true } -// NewSimple initializes a simple progress -func NewSimple(name string, total int) *Simple { - return &Simple{ +// NewIterative initializes an iterative progress +func NewIterative(name string, total int) *Iterative { + return &Iterative{ name: name, total: total, startTime: time.Now(), @@ -62,8 +66,8 @@ func NewSimple(name string, total int) *Simple { } func init() { - err := progress.Progresses.Register(simpleType, func(name string, total int) model.Progress { - return NewSimple(name, total) + err := progress.Progresses.Register(iterativeType, func(name string, total int) model.Progress { + return NewIterative(name, total) }) if err != nil { panic(err) diff --git a/plugin/progress/verbose.go b/plugin/progress/progressive.go similarity index 59% rename from plugin/progress/verbose.go rename to plugin/progress/progressive.go index 537c723..876940e 100644 --- a/plugin/progress/verbose.go +++ b/plugin/progress/progressive.go @@ -10,32 +10,32 @@ import ( "github.com/vbauerster/mpb/v7/decor" ) -const verboseType = "verbose" +const progressiveType = "progressive" const ( maxNameLength = 21 defaultWidth = 64 ) -// Verbose defines how a verbose progress should be executed -type Verbose struct { +// Progressive defines how a progressive progress should be executed +type Progressive struct { progress *mpb.Progress bar *mpb.Bar } -// Increment increments the progress -func (v *Verbose) Increment() { - v.bar.Increment() +// Increase inceases the progress by the specified num +func (v *Progressive) Increase(num int) { + v.bar.IncrBy(num) } // Wait finishes the progress -func (v *Verbose) Wait() { +func (v *Progressive) Wait() { v.bar.SetTotal(0, true) v.progress.Wait() } -// NewVerbose initializes a verbose progress -func NewVerbose(name string, total int) *Verbose { +// NewProgressive initializes a progressive progress +func NewProgressive(name string, total int) *Progressive { name = standardize(name) progress := mpb.New() @@ -51,25 +51,27 @@ func NewVerbose(name string, total int) *Verbose { decor.Elapsed(decor.ET_STYLE_MMSS, decor.WCSyncSpace), ), ) - return &Verbose{ + return &Progressive{ progress: progress, bar: bar, } } func standardize(input string) string { + extender := "..." if len(input) > maxNameLength { - input = input[:maxNameLength] + "..." + input = input[:maxNameLength] + extender } - if len(input) < maxNameLength+3 { - input = input + strings.Repeat(" ", maxNameLength+3-len(input)) + limit := maxNameLength + len(extender) + if len(input) < limit { + input = input + strings.Repeat(" ", limit-len(input)) } return input } func init() { - err := progress.Progresses.Register(verboseType, func(name string, total int) model.Progress { - return NewVerbose(name, total) + err := progress.Progresses.Register(progressiveType, func(name string, total int) model.Progress { + return NewProgressive(name, total) }) if err != nil { panic(err) diff --git a/recipe/loader.go b/recipe/loader.go index 4507b11..0db782c 100644 --- a/recipe/loader.go +++ b/recipe/loader.go @@ -8,20 +8,19 @@ import ( ) // Load loads recipe from the passed Reader with Decoder to decode -func Load(reader model.Reader, decode model.Decode) (*Recipe, model.Error) { - const defaultErrKey = "Load" +func Load(reader model.Reader, decode model.Decode) (*Recipe, error) { if reader == nil { - return nil, model.BuildError(defaultErrKey, errors.New("reader is nil")) + return nil, errors.New("reader is nil") } if decode == nil { - return nil, model.BuildError(defaultErrKey, errors.New("decode is nil")) + return nil, errors.New("decode is nil") } - data, err := reader.ReadOne() + data, err := reader.Read() if err != nil { return nil, err } if len(bytes.TrimSpace(data.Content)) == 0 { - return nil, model.BuildError(defaultErrKey, errors.New("content is empty")) + return nil, errors.New("content is empty") } output := &Recipe{} err = decode(data.Content, output) diff --git a/recipe/loader_test.go b/recipe/loader_test.go index 5a525f4..52b3956 100644 --- a/recipe/loader_test.go +++ b/recipe/loader_test.go @@ -12,13 +12,11 @@ import ( ) func TestLoad(t *testing.T) { - const defaultErrKey = "Load" - t.Run("should return nil and error if reader is nil", func(t *testing.T) { var reader model.Reader = nil var decode model.Decode - expectedErr := model.BuildError(defaultErrKey, errors.New("reader is nil")) + expectedErr := errors.New("reader is nil") actualRecipe, actualErr := recipe.Load(reader, decode) @@ -30,7 +28,7 @@ func TestLoad(t *testing.T) { reader := &mocks.Reader{} var decode model.Decode = nil - expectedErr := model.BuildError(defaultErrKey, errors.New("decode is nil")) + expectedErr := errors.New("decode is nil") actualRecipe, actualErr := recipe.Load(reader, decode) @@ -39,10 +37,10 @@ func TestLoad(t *testing.T) { }) t.Run("should return nil and error if read returns error", func(t *testing.T) { - readErr := model.BuildError(defaultErrKey, errors.New("read error")) + readErr := errors.New("read error") reader := &mocks.Reader{} - reader.On("ReadOne").Return(nil, readErr) - decode := func(c []byte, v interface{}) model.Error { + reader.On("Read").Return(nil, readErr) + decode := func(c []byte, v interface{}) error { return nil } @@ -56,12 +54,12 @@ func TestLoad(t *testing.T) { t.Run("should return nil and error if content is empty", func(t *testing.T) { reader := &mocks.Reader{} - reader.On("ReadOne").Return(&model.Data{}, nil) - decode := func(c []byte, v interface{}) model.Error { + reader.On("Read").Return(&model.Data{}, nil) + decode := func(c []byte, v interface{}) error { return nil } - expectedErr := model.BuildError(defaultErrKey, errors.New("content is empty")) + expectedErr := errors.New("content is empty") actualRecipe, actualErr := recipe.Load(reader, decode) @@ -71,11 +69,11 @@ func TestLoad(t *testing.T) { t.Run("should return nil and error if decode returns error", func(t *testing.T) { reader := &mocks.Reader{} - reader.On("ReadOne").Return(&model.Data{ + reader.On("Read").Return(&model.Data{ Content: []byte("message"), }, nil) - decodeErr := model.BuildError(defaultErrKey, errors.New("decode error")) - decode := func(c []byte, v interface{}) model.Error { + decodeErr := errors.New("decode error") + decode := func(c []byte, v interface{}) error { return decodeErr } @@ -89,10 +87,10 @@ func TestLoad(t *testing.T) { t.Run("should return data and nil if no error is encountered", func(t *testing.T) { reader := &mocks.Reader{} - reader.On("ReadOne").Return(&model.Data{ + reader.On("Read").Return(&model.Data{ Content: []byte("message"), }, nil) - decode := func(c []byte, v interface{}) model.Error { + decode := func(c []byte, v interface{}) error { return nil } diff --git a/recipe/recipe.go b/recipe/recipe.go index 18c46a8..908d3f6 100644 --- a/recipe/recipe.go +++ b/recipe/recipe.go @@ -17,11 +17,10 @@ type Resource struct { // Framework is a recipe on how and where to read the actual Framework data type Framework struct { - Name string `yaml:"name" validate:"required"` - Definitions []*Definition `yaml:"definitions"` - Schemas []*Schema `yaml:"schemas"` - Procedures []*Procedure `yaml:"procedures"` - OutputTargets []*OutputTarget `yaml:"output_targets"` + Name string `yaml:"name" validate:"required"` + Schemas []*Schema `yaml:"schemas"` + Definitions []*Definition `yaml:"definitions"` + Procedures []*Procedure `yaml:"procedures"` } // Definition is a recipe on how and where to read the actual Definition data @@ -41,21 +40,28 @@ type Function struct { // Schema is a recipe on how and where to read the actual Schema data type Schema struct { - Name string `yaml:"name" validate:"required"` - Type string `yaml:"type" validate:"required,oneof=dir file"` - Path string `yaml:"path" validate:"required"` + Name string `yaml:"name" validate:"required"` + Type string `yaml:"type" validate:"required,oneof=dir file"` + Path string `yaml:"path" validate:"required"` + Output *Output `yaml:"output"` } // Procedure is a recipe on how and where to read the actual Procedure data type Procedure struct { - Name string `yaml:"name" validate:"required"` - Type string `yaml:"type" validate:"required,oneof=dir file"` - Path string `yaml:"path" validate:"required"` - OutputIsError bool `yaml:"output_is_error"` + Name string `yaml:"name" validate:"required"` + Type string `yaml:"type" validate:"required,oneof=dir file"` + Path string `yaml:"path" validate:"required"` + Output *Output `yaml:"output"` +} + +// Output defines how the last procedure output is written +type Output struct { + TreatAs string `yaml:"treat_as" validate:"required,oneof=info warning error success"` + Targets []*Target `yaml:"targets" validate:"required,min=1"` } -// OutputTarget defines how an output is created -type OutputTarget struct { +// Target defines how an output is written to the targetted stream +type Target struct { Name string `yaml:"name" validate:"required"` Format string `yaml:"format" validate:"required,oneof=json yaml"` Type string `yaml:"type" validate:"required,eq=dir"` diff --git a/recipe/validator.go b/recipe/validator.go index 99e8fb3..49c2276 100644 --- a/recipe/validator.go +++ b/recipe/validator.go @@ -4,17 +4,13 @@ import ( "fmt" "strings" - "github.com/gojek/optimus-extension-valor/model" - "github.com/go-playground/validator/v10" ) -const defaultValidateKey = "Validate" - // Validate validates the recipe -func Validate(rcp *Recipe) model.Error { +func Validate(rcp *Recipe) error { if err := validator.New().Struct(rcp); err != nil { - return model.BuildError(defaultValidateKey, err) + return err } if err := validateAllResources(rcp.Resources); err != nil { return err @@ -22,7 +18,7 @@ func Validate(rcp *Recipe) model.Error { return validateAllFrameworks(rcp.Frameworks) } -func validateAllResources(rcps []*Resource) model.Error { +func validateAllResources(rcps []*Resource) error { nameEncountered := make(map[string]int) for _, resourceRcp := range rcps { if err := ValidateResource(resourceRcp); err != nil { @@ -37,17 +33,13 @@ func validateAllResources(rcps []*Resource) model.Error { } } if len(duplicateNames) > 0 { - return model.BuildError( - defaultValidateKey, - fmt.Errorf("duplicate resource recipe [%s]", - strings.Join(duplicateNames, ", "), - ), - ) + return fmt.Errorf("duplicate resource recipe [%s]", + strings.Join(duplicateNames, ", ")) } return nil } -func validateAllFrameworks(rcps []*Framework) model.Error { +func validateAllFrameworks(rcps []*Framework) error { nameEncountered := make(map[string]int) for _, frameworkRcp := range rcps { if err := ValidateFramework(frameworkRcp); err != nil { @@ -62,30 +54,24 @@ func validateAllFrameworks(rcps []*Framework) model.Error { } } if len(duplicateNames) > 0 { - return model.BuildError( - defaultValidateKey, - fmt.Errorf("duplicate framework recipe [%s]", - strings.Join(duplicateNames, ", "), - ), - ) + return fmt.Errorf("duplicate framework recipe [%s]", + strings.Join(duplicateNames, ", ")) } return nil } // ValidateResource validates the recipe for a Resource -func ValidateResource(resourceRcp *Resource) model.Error { - const defaultErrKey = "ValidateResource" +func ValidateResource(resourceRcp *Resource) error { if err := validator.New().Struct(resourceRcp); err != nil { - return model.BuildError(defaultErrKey, err) + return err } return nil } // ValidateFramework validates the recipe for a Framework -func ValidateFramework(frameworkRcp *Framework) model.Error { - const defaultErrKey = "ValidateFramework" +func ValidateFramework(frameworkRcp *Framework) error { if err := validator.New().Struct(frameworkRcp); err != nil { - return model.BuildError(defaultErrKey, err) + return err } return nil } diff --git a/recipe/validator_test.go b/recipe/validator_test.go index ade9ada..ab59c0d 100644 --- a/recipe/validator_test.go +++ b/recipe/validator_test.go @@ -4,15 +4,12 @@ import ( "errors" "testing" - "github.com/gojek/optimus-extension-valor/model" "github.com/gojek/optimus-extension-valor/recipe" "github.com/stretchr/testify/assert" ) func TestValidate(t *testing.T) { - const defaultErrKey = "Validate" - t.Run("should return error if validator returns error", func(t *testing.T) { var rcp *recipe.Recipe = nil @@ -74,7 +71,7 @@ func TestValidate(t *testing.T) { }, } - expectedErr := model.BuildError(defaultErrKey, errors.New("duplicate resource recipe [resource1]")) + expectedErr := errors.New("duplicate resource recipe [resource1]") actualErr := recipe.Validate(rcp) @@ -128,7 +125,7 @@ func TestValidate(t *testing.T) { }, } - expectedErr := model.BuildError(defaultErrKey, errors.New("duplicate framework recipe [evaluate1]")) + expectedErr := errors.New("duplicate framework recipe [evaluate1]") actualErr := recipe.Validate(rcp) @@ -160,8 +157,6 @@ func TestValidate(t *testing.T) { } func TestValidateResource(t *testing.T) { - const defaultErrKey = "ValidateResource" - t.Run("should return error if validator returns error", func(t *testing.T) { var rcp *recipe.Resource = nil @@ -186,8 +181,6 @@ func TestValidateResource(t *testing.T) { } func TestValidateFramework(t *testing.T) { - const defaultErrKey = "ValidateFramework" - t.Run("should return error if validator returns error", func(t *testing.T) { var rcp *recipe.Framework = nil diff --git a/registry/endec/decoder.go b/registry/endec/decoder.go index 557d7cf..4911dba 100644 --- a/registry/endec/decoder.go +++ b/registry/endec/decoder.go @@ -14,25 +14,23 @@ type DecodeFactory struct { } // Register registers a factory function for a type -func (d *DecodeFactory) Register(format string, fn model.Decode) model.Error { - const defaultErrKey = "Register" +func (d *DecodeFactory) Register(format string, fn model.Decode) error { if fn == nil { - return model.BuildError(defaultErrKey, errors.New("Decode is nil")) + return errors.New("Decode is nil") } format = strings.ToLower(format) if d.typeToFn[format] != nil { - return model.BuildError(defaultErrKey, fmt.Errorf("[%s] is already registered", format)) + return fmt.Errorf("[%s] is already registered", format) } d.typeToFn[format] = fn return nil } // Get gets a factory function based on a type -func (d *DecodeFactory) Get(format string) (model.Decode, model.Error) { - const defaultErrKey = "Get" +func (d *DecodeFactory) Get(format string) (model.Decode, error) { format = strings.ToLower(format) if d.typeToFn[format] == nil { - return nil, model.BuildError(defaultErrKey, fmt.Errorf("[%s] is not registered", format)) + return nil, fmt.Errorf("[%s] is not registered", format) } return d.typeToFn[format], nil } diff --git a/registry/endec/decoder_test.go b/registry/endec/decoder_test.go index 2cc70ea..968598b 100644 --- a/registry/endec/decoder_test.go +++ b/registry/endec/decoder_test.go @@ -27,7 +27,7 @@ func (d *DecodeFactorySuite) TestRegister() { d.Run("should return error fn is already registered", func() { factory := endec.NewDecodeFactory() format := "yaml" - var fn model.Decode = func(b []byte, i interface{}) model.Error { + var fn model.Decode = func(b []byte, i interface{}) error { return nil } factory.Register(format, fn) @@ -40,7 +40,7 @@ func (d *DecodeFactorySuite) TestRegister() { d.Run("should return nil if no error is found", func() { factory := endec.NewDecodeFactory() format := "yaml" - var fn model.Decode = func(b []byte, i interface{}) model.Error { + var fn model.Decode = func(b []byte, i interface{}) error { return nil } @@ -55,7 +55,7 @@ func (d *DecodeFactorySuite) TestGet() { factory := endec.NewDecodeFactory() yamlFormat := "yaml" jsonFormat := "json" - var fn model.Decode = func(b []byte, i interface{}) model.Error { + var fn model.Decode = func(b []byte, i interface{}) error { return nil } factory.Register(yamlFormat, fn) @@ -69,7 +69,7 @@ func (d *DecodeFactorySuite) TestGet() { d.Run("should return fn and nil type is found found", func() { factory := endec.NewDecodeFactory() yamlFormat := "yaml" - var fn model.Decode = func(b []byte, i interface{}) model.Error { + var fn model.Decode = func(b []byte, i interface{}) error { return nil } factory.Register(yamlFormat, fn) diff --git a/registry/endec/encoder.go b/registry/endec/encoder.go index 606587f..429b29b 100644 --- a/registry/endec/encoder.go +++ b/registry/endec/encoder.go @@ -14,25 +14,23 @@ type EncodeFactory struct { } // Register registers a factory function for a type -func (e *EncodeFactory) Register(_type string, fn model.Encode) model.Error { - const defaultErrKey = "Register" +func (e *EncodeFactory) Register(_type string, fn model.Encode) error { if fn == nil { - return model.BuildError(defaultErrKey, errors.New("Encode is nil")) + return errors.New("Encode is nil") } _type = strings.ToLower(_type) if e.typeToFn[_type] != nil { - return model.BuildError(defaultErrKey, fmt.Errorf("[%s] is already registered", _type)) + return fmt.Errorf("[%s] is already registered", _type) } e.typeToFn[_type] = fn return nil } // Get gets a factory function based on a type -func (e *EncodeFactory) Get(_type string) (model.Encode, model.Error) { - const defaultErrKey = "Get" +func (e *EncodeFactory) Get(_type string) (model.Encode, error) { _type = strings.ToLower(_type) if e.typeToFn[_type] == nil { - return nil, model.BuildError(defaultErrKey, fmt.Errorf("[%s] is not registered", _type)) + return nil, fmt.Errorf("[%s] is not registered", _type) } return e.typeToFn[_type], nil } diff --git a/registry/endec/encoder_test.go b/registry/endec/encoder_test.go index 386b911..395eb49 100644 --- a/registry/endec/encoder_test.go +++ b/registry/endec/encoder_test.go @@ -27,7 +27,7 @@ func (e *EncodeFactorySuite) TestRegister() { e.Run("should return error fn is already registered", func() { factory := endec.NewEncodeFactory() format := "yaml" - var fn model.Encode = func(i interface{}) ([]byte, model.Error) { + var fn model.Encode = func(i interface{}) ([]byte, error) { return nil, nil } factory.Register(format, fn) @@ -40,7 +40,7 @@ func (e *EncodeFactorySuite) TestRegister() { e.Run("should return nil if no error is found", func() { factory := endec.NewEncodeFactory() format := "yaml" - var fn model.Encode = func(i interface{}) ([]byte, model.Error) { + var fn model.Encode = func(i interface{}) ([]byte, error) { return nil, nil } @@ -55,7 +55,7 @@ func (e *EncodeFactorySuite) TestGet() { factory := endec.NewEncodeFactory() yamlFormat := "yaml" jsonFormat := "json" - var fn model.Encode = func(i interface{}) ([]byte, model.Error) { + var fn model.Encode = func(i interface{}) ([]byte, error) { return nil, nil } factory.Register(yamlFormat, fn) @@ -69,7 +69,7 @@ func (e *EncodeFactorySuite) TestGet() { e.Run("should return fn and nil type is found found", func() { factory := endec.NewEncodeFactory() yamlFormat := "yaml" - var fn model.Encode = func(i interface{}) ([]byte, model.Error) { + var fn model.Encode = func(i interface{}) ([]byte, error) { return nil, nil } factory.Register(yamlFormat, fn) diff --git a/registry/explorer/explorer.go b/registry/explorer/explorer.go new file mode 100644 index 0000000..5803115 --- /dev/null +++ b/registry/explorer/explorer.go @@ -0,0 +1,43 @@ +package explorer + +import ( + "errors" + "fmt" + + "github.com/gojek/optimus-extension-valor/model" +) + +// Explorers is a factory for explorer +var Explorers = NewFactory() + +// Factory is a factory for Explorer +type Factory struct { + typeToFn map[string]model.ExplorePath +} + +// Register registers an explorer based on the type +func (f *Factory) Register(_type string, fn model.ExplorePath) error { + if fn == nil { + return errors.New("Explorer is nil") + } + if f.typeToFn[_type] != nil { + return fmt.Errorf("[%s] is already registered", _type) + } + f.typeToFn[_type] = fn + return nil +} + +// Get gets an explorer based on a specified type +func (f *Factory) Get(_type string) (model.ExplorePath, error) { + if f.typeToFn[_type] == nil { + return nil, fmt.Errorf("[%s] is not registered", _type) + } + return f.typeToFn[_type], nil +} + +// NewFactory initializes factory Formatter +func NewFactory() *Factory { + return &Factory{ + typeToFn: make(map[string]model.ExplorePath), + } +} diff --git a/registry/explorer/explorer_test.go b/registry/explorer/explorer_test.go new file mode 100644 index 0000000..03021d6 --- /dev/null +++ b/registry/explorer/explorer_test.go @@ -0,0 +1,80 @@ +package explorer_test + +import ( + "testing" + + "github.com/gojek/optimus-extension-valor/model" + "github.com/gojek/optimus-extension-valor/registry/explorer" + + "github.com/stretchr/testify/suite" +) + +type FactorySuite struct { + suite.Suite +} + +func (f *FactorySuite) TestRegister() { + f.Run("should return error if fn is nil", func() { + factory := explorer.NewFactory() + _type := "file" + var exploreFn model.ExplorePath = nil + + actualErr := factory.Register(_type, exploreFn) + + f.NotNil(actualErr) + }) + + f.Run("should return error fn is already registered", func() { + factory := explorer.NewFactory() + _type := "file" + exploreFn := func(root string, filter func(string) bool) ([]string, error) { + return nil, nil + } + factory.Register(_type, exploreFn) + + actualErr := factory.Register(_type, exploreFn) + + f.NotNil(actualErr) + }) + + f.Run("should return nil if no error is found", func() { + factory := explorer.NewFactory() + _type := "file" + exploreFn := func(root string, filter func(string) bool) ([]string, error) { + return nil, nil + } + + actualErr := factory.Register(_type, exploreFn) + + f.Nil(actualErr) + }) +} + +func (f *FactorySuite) TestGet() { + f.Run("should return nil and error if _type is not found", func() { + factory := explorer.NewFactory() + _type := "file" + + actualFn, actualErr := factory.Get(_type) + + f.Nil(actualFn) + f.NotNil(actualErr) + }) + + f.Run("should return fn and nil if _type is found", func() { + factory := explorer.NewFactory() + _type := "file" + factory.Register(_type, func(root string, filter func(string) bool) ([]string, error) { + return nil, nil + }) + + actualFn, actualErr := factory.Get(_type) + + f.NotNil(actualFn) + f.Nil(actualErr) + }) +} + +func TestWriterFactorySuite(t *testing.T) { + suite.Run(t, &FactorySuite{}) +} diff --git a/registry/formatter/formatter.go b/registry/formatter/formatter.go index 5d765df..3f45795 100644 --- a/registry/formatter/formatter.go +++ b/registry/formatter/formatter.go @@ -17,15 +17,14 @@ type Factory struct { } // Register registers a factory function for a specified source and destination -func (f *Factory) Register(src, dest string, fn model.Format) model.Error { - const defaultErrKey = "Register" +func (f *Factory) Register(src, dest string, fn model.Format) error { if fn == nil { - return model.BuildError(defaultErrKey, errors.New("Format is nil")) + return errors.New("Format is nil") } src = strings.ToLower(src) dest = strings.ToLower(dest) if f.srcToDestToFn[src] != nil && f.srcToDestToFn[src][dest] != nil { - return model.BuildError(defaultErrKey, fmt.Errorf("[source: %s | target: %s] is already registered", src, dest)) + return fmt.Errorf("[source: %s | target: %s] is already registered", src, dest) } if f.srcToDestToFn[src] == nil { f.srcToDestToFn[src] = make(map[string]model.Format) @@ -35,12 +34,11 @@ func (f *Factory) Register(src, dest string, fn model.Format) model.Error { } // Get gets a factory function based on a specified source and destination -func (f *Factory) Get(src, dest string) (model.Format, model.Error) { - const defaultErrKey = "Get" +func (f *Factory) Get(src, dest string) (model.Format, error) { src = strings.ToLower(src) dest = strings.ToLower(dest) if f.srcToDestToFn[src] == nil || f.srcToDestToFn[src][dest] == nil { - return nil, model.BuildError(defaultErrKey, fmt.Errorf("[source: %s | target: %s] is not registered", src, dest)) + return nil, fmt.Errorf("[source: %s | target: %s] is not registered", src, dest) } return f.srcToDestToFn[src][dest], nil } diff --git a/registry/formatter/formatter_test.go b/registry/formatter/formatter_test.go index cb2f2ae..d50827b 100644 --- a/registry/formatter/formatter_test.go +++ b/registry/formatter/formatter_test.go @@ -29,7 +29,7 @@ func (f *FactorySuite) TestRegister() { factory := formatter.NewFactory() src := "json" dest := "yaml" - var fn model.Format = func(b []byte) ([]byte, model.Error) { + var fn model.Format = func(b []byte) ([]byte, error) { return nil, nil } factory.Register(src, dest, fn) @@ -43,7 +43,7 @@ func (f *FactorySuite) TestRegister() { factory := formatter.NewFactory() src := "json" dest := "yaml" - var fn model.Format = func(b []byte) ([]byte, model.Error) { + var fn model.Format = func(b []byte) ([]byte, error) { return nil, nil } @@ -58,7 +58,7 @@ func (f *FactorySuite) TestGet() { factory := formatter.NewFactory() src := "json" dest := "yaml" - var fn model.Format = func(b []byte) ([]byte, model.Error) { + var fn model.Format = func(b []byte) ([]byte, error) { return nil, nil } factory.Register(src, dest, fn) @@ -73,7 +73,7 @@ func (f *FactorySuite) TestGet() { factory := formatter.NewFactory() src := "json" dest := "yaml" - var fn model.Format = func(b []byte) ([]byte, model.Error) { + var fn model.Format = func(b []byte) ([]byte, error) { return nil, nil } factory.Register(src, dest, fn) diff --git a/registry/io/reader.go b/registry/io/reader.go index 7da42c9..c32fb43 100644 --- a/registry/io/reader.go +++ b/registry/io/reader.go @@ -11,7 +11,6 @@ import ( // ReaderFn is a getter for IO Reader instance type ReaderFn func( getPath model.GetPath, - filterPath model.FilterPath, postProcess model.PostProcess, ) model.Reader @@ -21,25 +20,23 @@ type ReaderFactory struct { } // Register registers a factory function for a type -func (r *ReaderFactory) Register(_type string, fn ReaderFn) model.Error { - const defaultErrKey = "Register" +func (r *ReaderFactory) Register(_type string, fn ReaderFn) error { if fn == nil { - return model.BuildError(defaultErrKey, errors.New("ReaderFn is nil")) + return errors.New("ReaderFn is nil") } _type = strings.ToLower(_type) if r.typeToFn[_type] != nil { - return model.BuildError(defaultErrKey, fmt.Errorf("[%s] is already registered", _type)) + return fmt.Errorf("[%s] is already registered", _type) } r.typeToFn[_type] = fn return nil } // Get gets a factory function based on a type -func (r *ReaderFactory) Get(_type string) (ReaderFn, model.Error) { - const defaultErrKey = "Get" +func (r *ReaderFactory) Get(_type string) (ReaderFn, error) { _type = strings.ToLower(_type) if r.typeToFn[_type] == nil { - return nil, model.BuildError(defaultErrKey, fmt.Errorf("[%s] is not registered", _type)) + return nil, fmt.Errorf("[%s] is not registered", _type) } return r.typeToFn[_type], nil } diff --git a/registry/io/reader_test.go b/registry/io/reader_test.go index 7eab9f5..258177b 100644 --- a/registry/io/reader_test.go +++ b/registry/io/reader_test.go @@ -27,7 +27,7 @@ func (r *ReaderFactorySuite) TestRegister() { r.Run("should return error fn is already registered", func() { factory := io.NewReaderFactory() _type := "file" - var fn io.ReaderFn = func(getPath model.GetPath, filterPath model.FilterPath, postProcess model.PostProcess) model.Reader { + var fn io.ReaderFn = func(getPath model.GetPath, postProcess model.PostProcess) model.Reader { return nil } factory.Register(_type, fn) @@ -40,7 +40,7 @@ func (r *ReaderFactorySuite) TestRegister() { r.Run("should return nil if no error is found", func() { factory := io.NewReaderFactory() _type := "file" - var fn io.ReaderFn = func(getPath model.GetPath, filterPath model.FilterPath, postProcess model.PostProcess) model.Reader { + var fn io.ReaderFn = func(getPath model.GetPath, postProcess model.PostProcess) model.Reader { return nil } @@ -54,7 +54,7 @@ func (r *ReaderFactorySuite) TestGet() { r.Run("should return nil and error type is not found", func() { factory := io.NewReaderFactory() _type := "file" - var fn io.ReaderFn = func(getPath model.GetPath, filterPath model.FilterPath, postProcess model.PostProcess) model.Reader { + var fn io.ReaderFn = func(getPath model.GetPath, postProcess model.PostProcess) model.Reader { return nil } factory.Register(_type, fn) @@ -68,7 +68,7 @@ func (r *ReaderFactorySuite) TestGet() { r.Run("should return fn and nil type is found found", func() { factory := io.NewReaderFactory() _type := "file" - var fn io.ReaderFn = func(getPath model.GetPath, filterPath model.FilterPath, postProcess model.PostProcess) model.Reader { + var fn io.ReaderFn = func(getPath model.GetPath, postProcess model.PostProcess) model.Reader { return nil } factory.Register(_type, fn) diff --git a/registry/io/writer.go b/registry/io/writer.go index f3310cd..063d309 100644 --- a/registry/io/writer.go +++ b/registry/io/writer.go @@ -8,31 +8,32 @@ import ( "github.com/gojek/optimus-extension-valor/model" ) +// WriterFn is a getter for IO Writer instance +type WriterFn func(treatment model.OutputTreatment) model.Writer + // WriterFactory is a factory for Writer type WriterFactory struct { - typeToFn map[string]model.Writer + typeToFn map[string]WriterFn } // Register registers a factory function for a type -func (w *WriterFactory) Register(_type string, fn model.Writer) model.Error { - const defaultErrKey = "Register" +func (w *WriterFactory) Register(_type string, fn WriterFn) error { if fn == nil { - return model.BuildError(defaultErrKey, errors.New("WriteFn is nil")) + return errors.New("WriteFn is nil") } _type = strings.ToLower(_type) if w.typeToFn[_type] != nil { - return model.BuildError(defaultErrKey, fmt.Errorf("[%s] is already registered", _type)) + return fmt.Errorf("[%s] is already registered", _type) } w.typeToFn[_type] = fn return nil } // Get gets a factory function based on a type -func (w *WriterFactory) Get(_type string) (model.Writer, model.Error) { - const defaultErrKey = "Get" +func (w *WriterFactory) Get(_type string) (WriterFn, error) { _type = strings.ToLower(_type) if w.typeToFn[_type] == nil { - return nil, model.BuildError(defaultErrKey, fmt.Errorf("[%s] is not registered", _type)) + return nil, fmt.Errorf("[%s] is not registered", _type) } return w.typeToFn[_type], nil } @@ -40,6 +41,6 @@ func (w *WriterFactory) Get(_type string) (model.Writer, model.Error) { // NewWriterFactory initializes factory Writer func NewWriterFactory() *WriterFactory { return &WriterFactory{ - typeToFn: make(map[string]model.Writer), + typeToFn: make(map[string]WriterFn), } } diff --git a/registry/io/writer_test.go b/registry/io/writer_test.go index 87ee307..13e240f 100644 --- a/registry/io/writer_test.go +++ b/registry/io/writer_test.go @@ -14,62 +14,72 @@ type WriterFactorySuite struct { suite.Suite } -func (r *WriterFactorySuite) TestRegister() { - r.Run("should return error if fn is nil", func() { +func (w *WriterFactorySuite) TestRegister() { + w.Run("should return error if fn is nil", func() { factory := io.NewWriterFactory() _type := "file" - var writer model.Writer = nil + var writerFn io.WriterFn = nil - actualErr := factory.Register(_type, writer) + actualErr := factory.Register(_type, writerFn) - r.NotNil(actualErr) + w.NotNil(actualErr) }) - r.Run("should return error fn is already registered", func() { + w.Run("should return error fn is already registered", func() { factory := io.NewWriterFactory() _type := "file" - var writer model.Writer = &mocks.Writer{} - factory.Register(_type, writer) + var writerFn io.WriterFn = func(model.OutputTreatment) model.Writer { + return &mocks.Writer{} + } + factory.Register(_type, writerFn) - actualErr := factory.Register(_type, writer) + actualErr := factory.Register(_type, writerFn) - r.NotNil(actualErr) + w.NotNil(actualErr) }) - r.Run("should return nil if no error is found", func() { + w.Run("should return nil if no error is found", func() { factory := io.NewWriterFactory() _type := "file" - var writer model.Writer = &mocks.Writer{} + var writerFn io.WriterFn = func(model.OutputTreatment) model.Writer { + return &mocks.Writer{} + } - actualErr := factory.Register(_type, writer) + actualErr := factory.Register(_type, writerFn) - r.Nil(actualErr) + w.Nil(actualErr) }) } -func (r *WriterFactorySuite) TestGet() { - r.Run("should return nil and error type is not found", func() { +func (w *WriterFactorySuite) TestGet() { + w.Run("should return nil and error type is not found", func() { factory := io.NewWriterFactory() _type := "file" - var writer model.Writer = &mocks.Writer{} - factory.Register(_type, writer) + var writerFn io.WriterFn = func(model.OutputTreatment) model.Writer { + return &mocks.Writer{} + } + factory.Register(_type, writerFn) + factory.Register(_type, writerFn) actualWriter, actualErr := factory.Get("dir") - r.Nil(actualWriter) - r.NotNil(actualErr) + w.Nil(actualWriter) + w.NotNil(actualErr) }) - r.Run("should return fn and nil type is found found", func() { + w.Run("should return fn and nil type is found found", func() { factory := io.NewWriterFactory() _type := "file" - var writer model.Writer = &mocks.Writer{} - factory.Register(_type, writer) + var writerFn io.WriterFn = func(model.OutputTreatment) model.Writer { + return &mocks.Writer{} + } + factory.Register(_type, writerFn) + factory.Register(_type, writerFn) actualWriter, actualErr := factory.Get(_type) - r.NotNil(actualWriter) - r.Nil(actualErr) + w.NotNil(actualWriter) + w.Nil(actualErr) }) } diff --git a/registry/progress/progress.go b/registry/progress/progress.go index 70d267d..120c2dc 100644 --- a/registry/progress/progress.go +++ b/registry/progress/progress.go @@ -1,8 +1,8 @@ package progress import ( + "errors" "fmt" - "strings" "github.com/gojek/optimus-extension-valor/model" ) @@ -16,22 +16,21 @@ type Factory struct { } // Register registers a factory function for a specified type -func (f *Factory) Register(_type string, fn model.NewProgress) model.Error { - const defaultErrKey = "Register" - _type = strings.ToLower(_type) +func (f *Factory) Register(_type string, fn model.NewProgress) error { + if fn == nil { + return errors.New("NewProgress is nil") + } if f.typeToFn[_type] != nil { - return model.BuildError(defaultErrKey, fmt.Errorf("[%s] is already registered", _type)) + return fmt.Errorf("[%s] is already registered", _type) } f.typeToFn[_type] = fn return nil } // Get gets a factory function based on a specified type -func (f *Factory) Get(_type string) (model.NewProgress, model.Error) { - const defaultErrKey = "Get" - _type = strings.ToLower(_type) +func (f *Factory) Get(_type string) (model.NewProgress, error) { if f.typeToFn[_type] == nil { - return nil, model.BuildError(defaultErrKey, fmt.Errorf("[%s] is not registered", _type)) + return nil, fmt.Errorf("[%s] is not registered", _type) } return f.typeToFn[_type], nil } diff --git a/registry/progress/progress_test.go b/registry/progress/progress_test.go new file mode 100644 index 0000000..d927d2a --- /dev/null +++ b/registry/progress/progress_test.go @@ -0,0 +1,85 @@ +package progress_test + +import ( + "testing" + + "github.com/gojek/optimus-extension-valor/model" + "github.com/gojek/optimus-extension-valor/registry/progress" + + "github.com/stretchr/testify/suite" +) + +type FactorySuite struct { + suite.Suite +} + +func (f *FactorySuite) TestRegister() { + f.Run("should return error if fn is nil", func() { + factory := progress.NewFactory() + _type := "progressive" + var fn model.NewProgress = nil + + actualErr := factory.Register(_type, fn) + + f.NotNil(actualErr) + }) + + f.Run("should return error fn is already registered", func() { + factory := progress.NewFactory() + _type := "progressive" + var fn model.NewProgress = func(name string, total int) model.Progress { + return nil + } + factory.Register(_type, fn) + + actualErr := factory.Register(_type, fn) + + f.NotNil(actualErr) + }) + + f.Run("should return nil if no error is found", func() { + factory := progress.NewFactory() + _type := "progressive" + var fn model.NewProgress = func(name string, total int) model.Progress { + return nil + } + + actualErr := factory.Register(_type, fn) + + f.Nil(actualErr) + }) +} + +func (f *FactorySuite) TestGet() { + f.Run("should return nil and error type is not found", func() { + factory := progress.NewFactory() + _type := "progressive" + var fn model.NewProgress = func(name string, total int) model.Progress { + return nil + } + factory.Register(_type, fn) + + actualFn, actualErr := factory.Get("file") + + f.Nil(actualFn) + f.NotNil(actualErr) + }) + + f.Run("should return fn and nil type is found found", func() { + factory := progress.NewFactory() + _type := "progressive" + var fn model.NewProgress = func(name string, total int) model.Progress { + return nil + } + factory.Register(_type, fn) + + actualFn, actualErr := factory.Get(_type) + + f.NotNil(actualFn) + f.Nil(actualErr) + }) +} + +func TestReaderFactorySuite(t *testing.T) { + suite.Run(t, &FactorySuite{}) +} diff --git a/valor.example.yaml b/valor.example.yaml index a33ab47..4abaa71 100644 --- a/valor.example.yaml +++ b/valor.example.yaml @@ -1,6 +1,6 @@ resources: - name: user_account - type: dir + type: file path: ./example/resource format: json framework_names: @@ -13,10 +13,16 @@ frameworks: type: file format: json path: ./example/schema/user_account_rule.json + output: + treat_as: error + targets: + - name: std_output + type: std + format: yaml definitions: - name: memberships format: json - type: dir + type: file path: ./example/definition function: type: file @@ -26,8 +32,9 @@ frameworks: type: file format: jsonnet path: ./example/procedure/enrich_user_account.jsonnet - output_is_error: false - output_targets: - - name: std_output - type: std - format: json + output: + treat_as: success + targets: + - name: std_output + type: std + format: yaml