refactor: rework flow for more efficiency and consistency

* refactor: re-structure evaluation execution to be more consistent with validation

* refactor: fix typo for profile.go file

* feat: allows for customise output treatment

* refactor: rename progress to be more intuitive

* refactor: rework major flow process to be more memory efficient

New Feature:
* introduces output treatment for schema and procedure
* user can configure output given by schema and procedure respectively
* a bit of color for each output treatment

Bug Fix:
* bug caused by multiple definitions set within a single framework

Reworked Feature:
* framework output is now replaced by output teatment in each schema and procedure
* 'dir' I/O is replaced by 'explorer' for more generic path exploration
* validator and evaluator now handles how to write output, depending on the treatment

Refactor:
* change progress type from 'simple' and 'verbose' to 'iterative' and 'progressive' respectively
* simplify Loader in `core` package to be more readable and easier to maintain
* simplify Execute method in Pipeline to be more readable and easier to maintain
* resource is now loaded only when needed to reduce memory consumption

Concept:
* error is divided into two state: execution error and business error
* execution error is thrown if there's any anomaly outside business, such error when reading file
* business error is thrown only if execution does not meet business specified by the user, such as schema execution result

* fix: error on unit testing

* docs: add log for better clarity on execution

* docs: update documentation to reflect the latest changes

* refactor: error model changed to be a structure for thread-safe execution

README.md

@@ -60,11 +60,12 @@ And done. Based on the recipe, the output will be printed to the std out, or in
 
 ```zsh
 ...
-{
-   "email": "[email protected]",
-   "is_active": true,
-   "membership": "premium"
-}
+---------------------------
+example/resource/valor.json
+---------------------------
+email: [email protected]
+is_active: true
+membership: premium
 ...
 ```
 
@@ -76,8 +77,8 @@ What Valor does is actually stated in the recipe file `valor.yaml`. Behind the s
 2. execute framework pointed by field `framework_names`
 3. in the framework, run validation based on `schemas`
 4. if no error is found, load the required definition under `definitions`
-5. if no error is found, execute proceduresstated under `procedures`
-6. if no error is found, write the result based on `output_targets`
+5. if no error is found, execute procedures stated under `procedures`
+6. if no error is found, write the result based on `output`
 
 The explanation here is quite brief. For further explanation, try checkout the documentation [here](#documentation).
 

cmd/execute.go

@@ -17,7 +17,7 @@ import (
 
 const (
 	defaultBatchSize    = 4
-	defaultProgressType = "verbose"
+	defaultProgressType = "progressive"
 )
 
 var (
@@ -30,10 +30,7 @@ func getExecuteCmd() *cobra.Command {
 		Use:   "execute",
 		Short: "Execute pipeline based on the specified recipe",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := executePipeline(recipePath, batchSize, progressType, nil); err != nil {
-				return errors.New(string(err.JSON()))
-			}
-			return nil
+			return executePipeline(recipePath, batchSize, progressType, nil)
 		},
 	}
 	runCmd.PersistentFlags().StringVarP(&recipePath, "recipe-path", "R", defaultRecipePath, "Path of the recipe file")
@@ -44,7 +41,7 @@ func getExecuteCmd() *cobra.Command {
 	return runCmd
 }
 
-func executePipeline(recipePath string, batchSize int, progressType string, enrich func(*recipe.Recipe) model.Error) model.Error {
+func executePipeline(recipePath string, batchSize int, progressType string, enrich func(*recipe.Recipe) error) error {
 	rcp, err := loadRecipe(recipePath, defaultRecipeType, defaultRecipeFormat)
 	if err != nil {
 		return err
@@ -62,14 +59,15 @@ func executePipeline(recipePath string, batchSize int, progressType string, enri
 		return err
 	}
 	evaluate := getEvaluate()
-	pipeline, err := core.NewPipeline(rcp, batchSize, evaluate, newProgress)
+	pipeline, err := core.NewPipeline(rcp, evaluate, batchSize, newProgress)
 	if err != nil {
 		return err
 	}
-	if err := pipeline.Execute(); err != nil {
-		return err
+	err = pipeline.Execute()
+	if e, ok := err.(*model.Error); ok {
+		return errors.New(string(e.JSON()))
 	}
-	return nil
+	return err
 }
 
 func getEvaluate() model.Evaluate {
@@ -79,32 +77,25 @@ func getEvaluate() model.Evaluate {
 	}
 }
 
-func loadRecipe(path, _type, format string) (*recipe.Recipe, model.Error) {
+func loadRecipe(path, _type, format string) (*recipe.Recipe, error) {
 	fnReader, err := io.Readers.Get(_type)
 	if err != nil {
 		return nil, err
 	}
 	getPath := func() string {
 		return path
 	}
-	filterPath := func(p string) bool {
-		return true
-	}
-	postProcess := func(p string, c []byte) (*model.Data, model.Error) {
+	postProcess := func(p string, c []byte) (*model.Data, error) {
 		return &model.Data{
 			Content: bytes.ToLower(c),
 			Path:    p,
 			Type:    format,
 		}, nil
 	}
-	reader := fnReader(getPath, filterPath, postProcess)
+	reader := fnReader(getPath, postProcess)
 	decode, err := endec.Decodes.Get(format)
 	if err != nil {
 		return nil, err
 	}
-	rcp, err := recipe.Load(reader, decode)
-	if err != nil {
-		return nil, err
-	}
-	return rcp, nil
+	return recipe.Load(reader, decode)
 }

cmd/profie.go → cmd/profile.go

@@ -1,7 +1,6 @@
 package cmd
 
 import (
-	"errors"
 	"fmt"
 	"os"
 
@@ -18,7 +17,7 @@ func getProfileCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			rcp, err := loadRecipe(recipePath, defaultRecipeType, defaultRecipeFormat)
 			if err != nil {
-				return errors.New(string(err.JSON()))
+				return err
 			}
 
 			fmt.Println("RESOURCE:")
@@ -66,8 +65,12 @@ func getFrameworkTable(rcp *recipe.Recipe) *tablewriter.Table {
 		for _, p := range f.Procedures {
 			table.Append([]string{f.Name, "procedure", p.Name})
 		}
-		for _, o := range f.OutputTargets {
-			table.Append([]string{f.Name, "output", o.Name})
+		for _, p := range f.Procedures {
+			if p.Output != nil {
+				for _, t := range p.Output.Targets {
+					table.Append([]string{f.Name, "output", t.Path})
+				}
+			}
 		}
 	}
 	return table

cmd/resource.go

@@ -28,18 +28,19 @@ func getResourceCmd() *cobra.Command {
 		Use:   "resource",
 		Short: "Execute pipeline for a specific resource",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			enrich := func(rcp *recipe.Recipe) model.Error {
+			enrich := func(rcp *recipe.Recipe) error {
 				return enrichRecipe(rcp, &resourceArg{
 					Name:   name,
 					Format: format,
 					Type:   _type,
 					Path:   path,
 				})
 			}
-			if err := executePipeline(recipePath, batchSize, progressType, enrich); err != nil {
-				return errors.New(string(err.JSON()))
+			err := executePipeline(recipePath, batchSize, progressType, enrich)
+			if e, ok := err.(*model.Error); ok {
+				return errors.New(string(e.JSON()))
 			}
-			return nil
+			return err
 		},
 	}
 	resourceCmd.Flags().StringVarP(&name, "name", "n", "", "name of the resource recipe to be used")
@@ -51,7 +52,7 @@ func getResourceCmd() *cobra.Command {
 	return resourceCmd
 }
 
-func enrichRecipe(rcp *recipe.Recipe, arg *resourceArg) model.Error {
+func enrichRecipe(rcp *recipe.Recipe, arg *resourceArg) error {
 	if arg.Name == "" {
 		return nil
 	}
@@ -71,12 +72,8 @@ func enrichRecipe(rcp *recipe.Recipe, arg *resourceArg) model.Error {
 			break
 		}
 	}
-	const defaultErrKey = "enrichRecipe"
 	if resourceRcp == nil {
-		return model.BuildError(
-			defaultErrKey,
-			fmt.Errorf("resource recipe [%s] is not found", arg.Name),
-		)
+		return fmt.Errorf("resource recipe [%s] is not found", arg.Name)
 	}
 	rcp.Resources = []*recipe.Resource{resourceRcp}
 	return nil