Merge pull request #578 from lanasalameh1/lanasalameh/add-report-flag…

…-conftest-verify

add --report flag to conftest verify

acceptance.bats

@@ -88,6 +88,66 @@
   [ "$status" -eq 1 ]
 }
 
+@test "Verify command has report flag - no failures" {
+    run ./conftest verify --policy ./examples/report/policy --policy ./examples/report/success --report fails
+    [ "$status" -eq 0 ]
+    [[ "${lines[0]}" =~ "data.main.test_no_missing_label: PASS" ]]
+    [[ "${lines[1]}" =~ "--------------------------------------------------------------------------------" ]]
+    [[ "${lines[2]}" =~ "PASS: 1/1" ]]
+}
+
+@test "Verify command does not support report flag with table output" {
+    run ./conftest verify --policy ./examples/report/policy -o table --report fails
+    [[ "$output" =~ "Error: report flag is supported with stdout only" ]]
+}
+
+@test "Verify command does not support report flag with tap output" {
+    run ./conftest verify --policy ./examples/report/policy -o tap --report fails
+    [[ "$output" =~ "Error: report flag is supported with stdout only" ]]
+}
+
+@test "Verify command does not support report flag with junit output" {
+    run ./conftest verify --policy ./examples/report/policy -o junit --report fails
+    [[ "$output" =~ "Error: report flag is supported with stdout only" ]]
+}
+
+@test "Verify command does not support report flag with json output" {
+    run ./conftest verify --policy ./examples/report/policy -o json --report fails
+    [[ "$output" =~ "Error: report flag is supported with stdout only" ]]
+}
+
+@test "Verify command has report flag - failure with report fails" {
+    run ./conftest verify --policy ./examples/report/policy --policy ./examples/report/fail --report fails
+    [ "$status" -eq 1 ]
+    [[ "$output" =~ "FAILURES" ]]
+    [[ "$output" =~ "data.main.test_missing_required_label_fail: FAIL" ]]
+    [[ "$output" =~ "Fail input.metadata.labels[\"app.kubernetes.io/name\"]" ]]
+    [[ "$output" =~ "SUMMARY" ]]
+    [[ "$output" =~ "FAIL: 1/1" ]]
+}
+
+@test "Verify command has report flag - failure with report notes" {
+    run ./conftest verify --policy ./examples/report/policy --policy ./examples/report/fail --report notes
+    [ "$status" -eq 1 ]
+    [[ "$output" =~ "FAILURES" ]]
+    [[ "$output" =~ "data.main.test_missing_required_label_fail: FAIL" ]]
+    [[ "$output" =~ "Note \"just testing notes flag\"" ]]
+    [[ "$output" =~ "SUMMARY" ]]
+    [[ "$output" =~ "FAIL: 1/1" ]]
+}
+
+@test "Verify command has report flag - failure with report full" {
+    run ./conftest verify --policy ./examples/report/policy --policy ./examples/report/fail --report full
+    [ "$status" -eq 1 ]
+    [[ "$output" =~ "FAILURES" ]]
+    [[ "$output" =~ "data.main.test_missing_required_label_fail: FAIL" ]]
+    [[ "$output" =~ "Eval input.metadata.labels[\"app.kubernetes.io/name\"]" ]]
+    [[ "$output" =~ "Fail input.metadata.labels[\"app.kubernetes.io/name\"]" ]]
+    [[ "$output" =~ "Note \"just testing notes flag\"" ]]
+    [[ "$output" =~ "SUMMARY" ]]
+    [[ "$output" =~ "FAIL: 1/1" ]]
+}
+
 @test "Has help flag" {
   run ./conftest --help
   [ "$status" -eq 0 ]

examples/report/fail/failing_test.rego

@@ -0,0 +1,19 @@
+package main
+
+no_violations {
+	count(deny) == 0
+}
+
+test_missing_required_label_fail {
+	input := {
+		"kind": "Deployment",
+		"metadata": {
+			"name": "sample",
+			"labels": {
+				"app.kubernetes.io/instance"
+			}
+		}
+	}
+
+	no_violations with input as input
+}

examples/report/policy/labels.rego

@@ -0,0 +1,15 @@
+package main
+
+name = input.metadata.name
+
+required_deployment_labels {
+	input.metadata.labels["app.kubernetes.io/name"]
+	input.metadata.labels["app.kubernetes.io/instance"]
+}
+
+deny[msg] {
+	input.kind = "Deployment"
+	not required_deployment_labels
+	trace("just testing notes flag")
+	msg = sprintf("%s must include Kubernetes recommended labels: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/#labels", [name])
+}

examples/report/success/success_test.rego

@@ -0,0 +1,20 @@
+package main
+
+no_violations {
+	count(deny) == 0
+}
+
+test_no_missing_label {
+	input := {
+		"kind": "Deployment",
+		"metadata": {
+			"name": "sample",
+			"labels": {
+				"app.kubernetes.io/name",
+				"app.kubernetes.io/instance"
+			}
+		}
+	}
+
+	no_violations with input as input
+}

internal/commands/verify.go

@@ -53,6 +53,11 @@ When debugging policies it can be useful to use a more verbose policy evaluation
 the output will include a detailed trace of how the policy was evaluated, e.g.
 
 	$ conftest verify --trace
+
+Use '--report' to get a report of the results with a summary. You can scope down to output full or notes or failed evaluation events {full|notes|fails}.
+	'full' - outputs all of the trace events
+	'notes' - outputs the trace events with 'trace(msg)' calls
+	'fails' - outputs the trace events of the failed queries
 `
 
 // NewVerifyCommand creates a new verify command which allows users
@@ -63,7 +68,7 @@ func NewVerifyCommand(ctx context.Context) *cobra.Command {
 		Short: "Verify Rego unit tests",
 		Long:  verifyDesc,
 		PreRunE: func(cmd *cobra.Command, args []string) error {
-			flagNames := []string{"data", "no-color", "output", "policy", "trace"}
+			flagNames := []string{"data", "no-color", "output", "policy", "trace", "report"}
 			for _, name := range flagNames {
 				if err := viper.BindPFlag(name, cmd.Flags().Lookup(name)); err != nil {
 					return fmt.Errorf("bind flag: %w", err)
@@ -78,14 +83,26 @@ func NewVerifyCommand(ctx context.Context) *cobra.Command {
 				return fmt.Errorf("unmarshal parameters: %w", err)
 			}
 
-			results, err := runner.Run(ctx)
+			results, raw, err := runner.Run(ctx)
 			if err != nil {
 				return fmt.Errorf("running verification: %w", err)
 			}
 
 			outputter := output.Get(runner.Output, output.Options{NoColor: runner.NoColor, Tracing: runner.Trace, ShowSkipped: true})
-			if err := outputter.Output(results); err != nil {
-				return fmt.Errorf("output results: %w", err)
+
+			if runner.IsReportOptionOn() {
+				// report currently available with stdout only
+				if runner.Output != output.OutputStandard {
+					return fmt.Errorf("report flag is supported with stdout only")
+				}
+
+				if err := outputter.Report(raw, runner.Report); err != nil {
+					return fmt.Errorf("report results: %w", err)
+				}
+			} else {
+				if err := outputter.Output(results); err != nil {
+					return fmt.Errorf("output results: %w", err)
+				}
 			}
 
 			exitCode := output.ExitCode(results)
@@ -99,6 +116,7 @@ func NewVerifyCommand(ctx context.Context) *cobra.Command {
 
 	cmd.Flags().Bool("no-color", false, "Disable color when printing")
 	cmd.Flags().Bool("trace", false, "Enable more verbose trace output for Rego queries")
+	cmd.Flags().String("report", "", "Shows output for Rego queries as a report with summary. Available options are {full|notes|fails}.")
 
 	cmd.Flags().StringP("output", "o", output.OutputStandard, fmt.Sprintf("Output format for conftest results - valid options are: %s", output.Outputs()))
 

internal/runner/verify.go

@@ -20,31 +20,50 @@ type VerifyRunner struct {
 	Output  string
 	NoColor bool `mapstructure:"no-color"`
 	Trace   bool
+	Report  string
+}
+
+const (
+	ReportFull  = "full"
+	ReportNotes = "notes"
+	ReportFails = "fails"
+)
+
+func (r *VerifyRunner) IsReportOptionOn() bool {
+	return r.Report == ReportFull ||
+		r.Report == ReportNotes ||
+		r.Report == ReportFails
+
 }
 
 // Run executes the Rego tests for the given policies.
-func (r *VerifyRunner) Run(ctx context.Context) ([]output.CheckResult, error) {
+func (r *VerifyRunner) Run(ctx context.Context) ([]output.CheckResult, []*tester.Result, error) {
 	engine, err := policy.LoadWithData(ctx, r.Policy, r.Data)
 	if err != nil {
-		return nil, fmt.Errorf("load: %w", err)
+		return nil, nil, fmt.Errorf("load: %w", err)
 	}
 
-	if r.Trace {
+	// Traces should be enabled when Trace or Report options are on
+	enableTracing := r.Trace || r.IsReportOptionOn()
+
+	if enableTracing {
 		engine.EnableTracing()
 	}
 
-	runner := tester.NewRunner().SetCompiler(engine.Compiler()).SetStore(engine.Store()).SetModules(engine.Modules()).EnableTracing(r.Trace).SetRuntime(engine.Runtime())
+	runner := tester.NewRunner().SetCompiler(engine.Compiler()).SetStore(engine.Store()).SetModules(engine.Modules()).EnableTracing(enableTracing).SetRuntime(engine.Runtime())
 	ch, err := runner.RunTests(ctx, nil)
 	if err != nil {
-		return nil, fmt.Errorf("running tests: %w", err)
+		return nil, nil, fmt.Errorf("running tests: %w", err)
 	}
 
 	var results []output.CheckResult
+	var rawResults []*tester.Result
 	for result := range ch {
 		if result.Error != nil {
-			return nil, fmt.Errorf("run test: %w", result.Error)
+			return nil, nil, fmt.Errorf("run test: %w", result.Error)
 		}
 
+		rawResults = append(rawResults, result)
 		buf := new(bytes.Buffer)
 		topdown.PrettyTrace(buf, result.Trace)
 		var traces []string
@@ -80,5 +99,5 @@ func (r *VerifyRunner) Run(ctx context.Context) ([]output.CheckResult, error) {
 		results = append(results, checkResult)
 	}
 
-	return results, nil
+	return results, rawResults, nil
 }

output/json.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+
+	"github.com/open-policy-agent/opa/tester"
 )
 
 // JSON represents an Outputter that outputs
@@ -45,3 +47,7 @@ func (j *JSON) Output(results []CheckResult) error {
 	fmt.Fprintln(j.Writer, out.String())
 	return nil
 }
+
+func (j *JSON) Report(results []*tester.Result, flag string) error {
+	return fmt.Errorf("report is not supported in JSON output")
+}

output/junit.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/jstemmer/go-junit-report/formatter"
 	"github.com/jstemmer/go-junit-report/parser"
+	"github.com/open-policy-agent/opa/tester"
 )
 
 // JUnit represents an Outputter that outputs
@@ -93,3 +94,7 @@ func getTestName(fileName string, namespace string, message string) string {
 
 	return fmt.Sprintf("%s - %s", fileName, namespace)
 }
+
+func (j *JUnit) Report(results []*tester.Result, flag string) error {
+	return fmt.Errorf("report is not supported in JUnit output")
+}

output/output.go

@@ -1,11 +1,16 @@
 package output
 
-import "os"
+import (
+	"os"
+
+	"github.com/open-policy-agent/opa/tester"
+)
 
 // Outputter controls how results of an evaluation will
 // be recorded and reported to the end user.
 type Outputter interface {
 	Output([]CheckResult) error
+	Report([]*tester.Result, string) error
 }
 
 // Options represents the options available when configuring

output/standard.go

@@ -3,8 +3,12 @@ package output
 import (
 	"fmt"
 	"io"
+	"os"
 
 	"github.com/logrusorgru/aurora"
+	"github.com/open-policy-agent/opa/tester"
+	"github.com/open-policy-agent/opa/topdown"
+	"github.com/open-policy-agent/opa/topdown/lineage"
 )
 
 // Standard represents an Outputter that outputs
@@ -164,3 +168,47 @@ func (s *Standard) outputTrace(results []CheckResult, colorizer aurora.Aurora) {
 		}
 	}
 }
+
+// outputs results as a report - similar to OPA test output
+func (s *Standard) Report(results []*tester.Result, flag string) error {
+	reporter := tester.PrettyReporter{
+		Verbose:     true,
+		Output:      os.Stdout,
+		FailureLine: true}
+
+	dup := make(chan *tester.Result)
+
+	go func() {
+		defer close(dup)
+		for i := 0; i < len(results); i++ {
+			results[i].Trace = filterTrace(results[i].Trace, flag)
+			dup <- results[i]
+		}
+	}()
+
+	if err := reporter.Report(dup); err != nil {
+		return fmt.Errorf("report results: %w", err)
+	}
+	return nil
+}
+
+// Filter traces - returns only failed traces
+func filterTrace(trace []*topdown.Event, flag string) []*topdown.Event {
+	if flag == "full" {
+		return trace
+	}
+	ops := map[topdown.Op]struct{}{}
+
+	if flag == "fails" {
+		ops[topdown.FailOp] = struct{}{}
+	}
+
+	if flag == "notes" {
+		ops[topdown.NoteOp] = struct{}{}
+	}
+
+	return lineage.Filter(trace, func(event *topdown.Event) bool {
+		_, relevant := ops[event.Op]
+		return relevant
+	})
+}

output/table.go

@@ -1,9 +1,11 @@
 package output
 
 import (
+	"fmt"
 	"io"
 
 	"github.com/olekukonko/tablewriter"
+	"github.com/open-policy-agent/opa/tester"
 )
 
 // Table represents an Outputter that outputs
@@ -56,3 +58,7 @@ func (t *Table) Output(checkResults []CheckResult) error {
 
 	return nil
 }
+
+func (t *Table) Report(results []*tester.Result, flag string) error {
+	return fmt.Errorf("report is not supported in table output")
+}

output/tap.go

@@ -3,6 +3,8 @@ package output
 import (
 	"fmt"
 	"io"
+
+	"github.com/open-policy-agent/opa/tester"
 )
 
 // TAP represents an Outputter that outputs
@@ -85,3 +87,7 @@ func (t *TAP) Output(checkResults []CheckResult) error {
 
 	return nil
 }
+
+func (t *TAP) Report(results []*tester.Result, flag string) error {
+	return fmt.Errorf("report is not supported in TAP output")
+}