From ce8df0b994dbdc949e2f1141185a7ff498772c2c Mon Sep 17 00:00:00 2001 From: Ahmed Elsabbahy Date: Mon, 26 Oct 2015 09:01:33 -0400 Subject: [PATCH 1/4] Cleanup validate/testResults in prep for json output format --- outputs/outputs.go | 22 ++++++++++++++++++++-- resource/validate.go | 25 +++++++++++-------------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/outputs/outputs.go b/outputs/outputs.go index becce16f3..80fc9ab83 100644 --- a/outputs/outputs.go +++ b/outputs/outputs.go @@ -34,13 +34,13 @@ func humanizeResult(r resource.TestResult) string { if r.Result { return green("%s: %s: %s: all expectations found: [%s]", r.ResourceType, r.Title, r.Property, strings.Join(r.Expected, ", ")) } else { - return red("%s: %s: %s: expectations not found [%s]", r.ResourceType, r.Title, r.Property, strings.Join(r.Expected, ", ")) + return red("%s: %s: %s: expectations not found [%s]", r.ResourceType, r.Title, r.Property, strings.Join(subtractSlice(r.Expected, r.Found), ", ")) } case resource.Contains: if r.Result { return green("%s: %s: %s: all patterns found: [%s]", r.ResourceType, r.Title, r.Property, strings.Join(r.Expected, ", ")) } else { - return red("%s: %s: %s: patterns not found: [%s]", r.ResourceType, r.Title, r.Property, strings.Join(r.Expected, ", ")) + return red("%s: %s: %s: patterns not found: [%s]", r.ResourceType, r.Title, r.Property, strings.Join(subtractSlice(r.Expected, r.Found), ", ")) } default: return red("Unexpected type %d", r.TestType) @@ -85,3 +85,21 @@ func GetOutputer(name string) Outputer { } return outputers[name] } + +func subtractSlice(x, y []string) []string { + m := make(map[string]bool) + + for _, y := range y { + m[y] = true + } + + var ret []string + for _, x := range x { + if m[x] { + continue + } + ret = append(ret, x) + } + + return ret +} diff --git a/resource/validate.go b/resource/validate.go index 871e5dc78..091c874c6 100644 --- a/resource/validate.go +++ b/resource/validate.go @@ -64,7 +64,7 @@ func ValidateValues(res IDer, property string, expectedValues []string, method f TestType: Values, Title: title, Property: property, - Expected: bad, + Expected: expectedValues, Found: foundValues, Duration: time.Now().Sub(startTime), } @@ -247,7 +247,10 @@ func ValidateContains(res IDer, property string, expectedValues []string, method i := 0 for _, pat := range notfound { if pat.Match(line) { - found = append(found, pat) + // Found it, but wasn't supposed to, don't mark it as found, but remove it from search + if !pat.Inverse() { + found = append(found, pat) + } continue } notfound[i] = pat @@ -270,29 +273,22 @@ func ValidateContains(res IDer, property string, expectedValues []string, method } } - var bad []patternMatcher + // Didn't find it, but we didn't want to.. so we mark it as found for _, pat := range notfound { - // pat.Pattern() == "" is for empty io.Reader - if pat.Inverse() || pat.Pattern() == "" { - continue - } - bad = append(bad, pat) - } - - for _, pat := range found { if pat.Inverse() { - bad = append(bad, pat) + found = append(found, pat) } } - if len(bad) > 0 { + if !reflect.DeepEqual(sliceToPatterns(expectedValues), found) { return TestResult{ Result: false, ResourceType: typs, TestType: Contains, Title: title, Property: property, - Expected: patternsToSlice(bad), + Expected: expectedValues, + Found: patternsToSlice(found), Duration: time.Now().Sub(startTime), } } @@ -303,6 +299,7 @@ func ValidateContains(res IDer, property string, expectedValues []string, method Title: title, Property: property, Expected: expectedValues, + Found: patternsToSlice(found), Duration: time.Now().Sub(startTime), } From dd0e3e9b98c7fc97d2e04ea0565254b95be132e3 Mon Sep 17 00:00:00 2001 From: Ahmed Elsabbahy Date: Mon, 26 Oct 2015 11:30:11 -0400 Subject: [PATCH 2/4] Fix code to pass tests Fix pattern order causeing test to fail Fix empty file/command causing test to fail --- resource/validate.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/resource/validate.go b/resource/validate.go index 091c874c6..8a3cd6ac1 100644 --- a/resource/validate.go +++ b/resource/validate.go @@ -273,14 +273,15 @@ func ValidateContains(res IDer, property string, expectedValues []string, method } } - // Didn't find it, but we didn't want to.. so we mark it as found for _, pat := range notfound { - if pat.Inverse() { + // Didn't find it, but we didn't want to.. so we mark it as found + // Empty pattern should match even if input to scanner is empty + if pat.Inverse() || pat.Pattern() == "" { found = append(found, pat) } } - if !reflect.DeepEqual(sliceToPatterns(expectedValues), found) { + if len(expectedValues) != len(found) { return TestResult{ Result: false, ResourceType: typs, From 06b393e534a16733790d0733c5fc23665e9dede7 Mon Sep 17 00:00:00 2001 From: Ahmed Elsabbahy Date: Mon, 26 Oct 2015 15:36:20 -0400 Subject: [PATCH 3/4] Use red color for errors in tests --- outputs/outputs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/outputs/outputs.go b/outputs/outputs.go index 80fc9ab83..df8a4e4a6 100644 --- a/outputs/outputs.go +++ b/outputs/outputs.go @@ -20,7 +20,7 @@ var red = color.New(color.FgRed).SprintfFunc() func humanizeResult(r resource.TestResult) string { if r.Err != nil { - return fmt.Sprintf("%s: %s: Error: %s", r.Title, r.Property, r.Err) + return red("%s: %s: Error: %s", r.Title, r.Property, r.Err) } switch r.TestType { From e202f7e74802c456e94d7617c69f3f8dc86b683e Mon Sep 17 00:00:00 2001 From: Ahmed Elsabbahy Date: Mon, 26 Oct 2015 17:26:21 -0400 Subject: [PATCH 4/4] Add --format json --- outputs/documentation.go | 10 +++++---- outputs/json.go | 46 +++++++++++++++++++++++++++++++++++++++ outputs/outputs.go | 6 ++--- outputs/rspecish.go | 13 ++++++----- resource/validate.go | 22 +++++++++---------- resource/validate_test.go | 24 ++++++++++---------- 6 files changed, 85 insertions(+), 36 deletions(-) create mode 100644 outputs/json.go diff --git a/outputs/documentation.go b/outputs/documentation.go index 2e0460892..a44d06de5 100644 --- a/outputs/documentation.go +++ b/outputs/documentation.go @@ -14,7 +14,7 @@ func (r Documentation) Output(results <-chan []resource.TestResult) (hasFail boo var failed []resource.TestResult for resultGroup := range results { for _, testResult := range resultGroup { - if testResult.Result { + if testResult.Successful { fmt.Println(humanizeResult(testResult)) testCount++ } else { @@ -26,18 +26,20 @@ func (r Documentation) Output(results <-chan []resource.TestResult) (hasFail boo fmt.Println("") } + fmt.Print("\n") if len(failed) > 0 { - color.Red("\nFailures:") + color.Red("Failures:") for _, testResult := range failed { fmt.Println(humanizeResult(testResult)) } + fmt.Print("\n") } if len(failed) > 0 { - color.Red("\nCount: %d failed: %d\n", testCount, len(failed)) + color.Red("Count: %d failed: %d\n", testCount, len(failed)) return true } - color.Green("\nCount: %d failed: %d\n", testCount, len(failed)) + color.Green("Count: %d failed: %d\n", testCount, len(failed)) return false } diff --git a/outputs/json.go b/outputs/json.go new file mode 100644 index 000000000..c58f8537f --- /dev/null +++ b/outputs/json.go @@ -0,0 +1,46 @@ +package outputs + +import ( + "encoding/json" + "fmt" + + "github.com/aelsabbahy/goss/resource" +) + +type Json struct{} + +func (r Json) Output(results <-chan []resource.TestResult) (hasFail bool) { + testCount := 0 + failed := 0 + var resultsOut []resource.TestResult + for resultGroup := range results { + for _, testResult := range resultGroup { + if !testResult.Successful { + failed++ + } + resultsOut = append(resultsOut, testResult) + testCount++ + } + } + + summary := make(map[string]interface{}) + summary["test_count"] = testCount + summary["failed_count"] = failed + + out := make(map[string]interface{}) + out["results"] = resultsOut + out["summary"] = summary + + j, _ := json.MarshalIndent(out, "", " ") + fmt.Println(string(j)) + + if failed > 0 { + return true + } + + return false +} + +func init() { + RegisterOutputer("json", &Json{}) +} diff --git a/outputs/outputs.go b/outputs/outputs.go index df8a4e4a6..88d8c19d9 100644 --- a/outputs/outputs.go +++ b/outputs/outputs.go @@ -25,19 +25,19 @@ func humanizeResult(r resource.TestResult) string { switch r.TestType { case resource.Value: - if r.Result { + if r.Successful { return green("%s: %s: %s: matches expectation: %s", r.ResourceType, r.Title, r.Property, r.Expected) } else { return red("%s: %s: %s: doesn't match, expect: %s found: %s", r.ResourceType, r.Title, r.Property, r.Expected, r.Found) } case resource.Values: - if r.Result { + if r.Successful { return green("%s: %s: %s: all expectations found: [%s]", r.ResourceType, r.Title, r.Property, strings.Join(r.Expected, ", ")) } else { return red("%s: %s: %s: expectations not found [%s]", r.ResourceType, r.Title, r.Property, strings.Join(subtractSlice(r.Expected, r.Found), ", ")) } case resource.Contains: - if r.Result { + if r.Successful { return green("%s: %s: %s: all patterns found: [%s]", r.ResourceType, r.Title, r.Property, strings.Join(r.Expected, ", ")) } else { return red("%s: %s: %s: patterns not found: [%s]", r.ResourceType, r.Title, r.Property, strings.Join(subtractSlice(r.Expected, r.Found), ", ")) diff --git a/outputs/rspecish.go b/outputs/rspecish.go index e062e80f0..737ec5d60 100644 --- a/outputs/rspecish.go +++ b/outputs/rspecish.go @@ -14,29 +14,30 @@ func (r Rspecish) Output(results <-chan []resource.TestResult) (hasFail bool) { var failed []resource.TestResult for resultGroup := range results { for _, testResult := range resultGroup { - if testResult.Result { + if testResult.Successful { fmt.Printf(green(".")) - testCount++ } else { fmt.Printf(red("F")) failed = append(failed, testResult) - testCount++ } + testCount++ } } + fmt.Print("\n\n") if len(failed) > 0 { - color.Red("\n\nFailures:") + color.Red("Failures:") for _, testResult := range failed { fmt.Println(humanizeResult(testResult)) } + fmt.Print("\n") } if len(failed) > 0 { - color.Red("\n\nCount: %d failed: %d\n", testCount, len(failed)) + color.Red("Count: %d failed: %d\n", testCount, len(failed)) return true } - color.Green("\n\nCount: %d failed: %d\n", testCount, len(failed)) + color.Green("Count: %d failed: %d\n", testCount, len(failed)) return false } diff --git a/resource/validate.go b/resource/validate.go index 8a3cd6ac1..236bc8223 100644 --- a/resource/validate.go +++ b/resource/validate.go @@ -17,7 +17,7 @@ const ( ) type TestResult struct { - Result bool + Successful bool Title string ResourceType string TestType int @@ -36,7 +36,7 @@ func ValidateValues(res IDer, property string, expectedValues []string, method f foundValues, err := method() if err != nil { return TestResult{ - Result: false, + Successful: false, ResourceType: typs, TestType: Values, Title: title, @@ -59,7 +59,7 @@ func ValidateValues(res IDer, property string, expectedValues []string, method f if len(bad) > 0 { return TestResult{ - Result: false, + Successful: false, ResourceType: typs, TestType: Values, Title: title, @@ -70,7 +70,7 @@ func ValidateValues(res IDer, property string, expectedValues []string, method f } } return TestResult{ - Result: true, + Successful: true, ResourceType: typs, TestType: Values, Title: title, @@ -90,7 +90,7 @@ func ValidateValue(res IDer, property string, expectedValue interface{}, method foundValue, err := method() if err != nil { return TestResult{ - Result: false, + Successful: false, ResourceType: typs, TestType: Value, Title: title, @@ -102,7 +102,7 @@ func ValidateValue(res IDer, property string, expectedValue interface{}, method if expectedValue == foundValue { return TestResult{ - Result: true, + Successful: true, ResourceType: typs, TestType: Value, Title: title, @@ -114,7 +114,7 @@ func ValidateValue(res IDer, property string, expectedValue interface{}, method } return TestResult{ - Result: false, + Successful: false, ResourceType: typs, TestType: Value, Title: title, @@ -229,7 +229,7 @@ func ValidateContains(res IDer, property string, expectedValues []string, method fh, err := method() if err != nil { return TestResult{ - Result: false, + Successful: false, ResourceType: typs, TestType: Contains, Title: title, @@ -263,7 +263,7 @@ func ValidateContains(res IDer, property string, expectedValues []string, method } if err := scanner.Err(); err != nil { return TestResult{ - Result: false, + Successful: false, ResourceType: typs, TestType: Contains, Title: title, @@ -283,7 +283,7 @@ func ValidateContains(res IDer, property string, expectedValues []string, method if len(expectedValues) != len(found) { return TestResult{ - Result: false, + Successful: false, ResourceType: typs, TestType: Contains, Title: title, @@ -294,7 +294,7 @@ func ValidateContains(res IDer, property string, expectedValues []string, method } } return TestResult{ - Result: true, + Successful: true, ResourceType: typs, TestType: Contains, Title: title, diff --git a/resource/validate_test.go b/resource/validate_test.go index 38f934584..f4d27e56a 100644 --- a/resource/validate_test.go +++ b/resource/validate_test.go @@ -32,8 +32,8 @@ func TestValidateValue(t *testing.T) { return c.in2, nil } got := ValidateValue(&FakeIDer{""}, "", c.in, inFunc) - if got.Result != c.want { - t.Errorf("%+v: got %v, want %v", c, got.Result, c.want) + if got.Successful != c.want { + t.Errorf("%+v: got %v, want %v", c, got.Successful, c.want) } } } @@ -44,8 +44,8 @@ func TestValidateValueErr(t *testing.T) { return c.in2, fmt.Errorf("some err") } got := ValidateValue(&FakeIDer{""}, "", c.in, inFunc) - if got.Result != false { - t.Errorf("%+v: got %v, want %v", c, got.Result, false) + if got.Successful != false { + t.Errorf("%+v: got %v, want %v", c, got.Successful, false) } } } @@ -75,8 +75,8 @@ func TestValidateValues(t *testing.T) { return c.in2, nil } got := ValidateValues(&FakeIDer{""}, "", c.in, inFunc) - if got.Result != c.want { - t.Errorf("%+v: got %v, want %v", c, got.Result, c.want) + if got.Successful != c.want { + t.Errorf("%+v: got %v, want %v", c, got.Successful, c.want) } } } @@ -87,8 +87,8 @@ func TestValidateValuesErr(t *testing.T) { return c.in2, fmt.Errorf("some err") } got := ValidateValues(&FakeIDer{""}, "", c.in, inFunc) - if got.Result != false { - t.Errorf("%+v: got %v, want %v", c, got.Result, false) + if got.Successful != false { + t.Errorf("%+v: got %v, want %v", c, got.Successful, false) } } } @@ -115,8 +115,8 @@ func TestValidateContains(t *testing.T) { return reader, nil } got := ValidateContains(&FakeIDer{""}, "", c.in, inFunc) - if got.Result != c.want { - t.Errorf("%+v: got %v, want %v", c, got.Result, c.want) + if got.Successful != c.want { + t.Errorf("%+v: got %v, want %v", c, got.Successful, c.want) } } } @@ -128,8 +128,8 @@ func TestValidateContainsErr(t *testing.T) { return reader, fmt.Errorf("some err") } got := ValidateContains(&FakeIDer{""}, "", c.in, inFunc) - if got.Result != false { - t.Errorf("%+v: got %v, want %v", c, got.Result, false) + if got.Successful != false { + t.Errorf("%+v: got %v, want %v", c, got.Successful, false) } } }