From 2a6af689b1b0a9d8d663abeedc8dd69f85952aa2 Mon Sep 17 00:00:00 2001 From: James Kwon <96548424+hongil0316@users.noreply.github.com> Date: Mon, 11 Nov 2024 20:14:52 -0500 Subject: [PATCH] Opa EvalWithOutput --- modules/opa/eval.go | 54 +++++++++++--- modules/opa/eval_test.go | 155 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 10 deletions(-) create mode 100644 modules/opa/eval_test.go diff --git a/modules/opa/eval.go b/modules/opa/eval.go index 0d5de5beb..d2c01d013 100644 --- a/modules/opa/eval.go +++ b/modules/opa/eval.go @@ -61,12 +61,41 @@ func Eval(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resu // opa eval -i $JSONFile -d $RulePath $ResultQuery // // This will asynchronously run OPA on each file concurrently using goroutines. -func EvalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) error { +// This will fail the test if any one of the files failed. +// For each file, the output will be returned on the outputs slice. +func EvalWithOutput(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) (outputs []string) { + outputs, err := EvalWithOutputE(t, options, jsonFilePaths, resultQuery) + require.NoError(t, err) + return +} + +// EvalE runs `opa eval` on the given JSON files using the configured policy file and result query. Translates to: +// +// opa eval -i $JSONFile -d $RulePath $ResultQuery +// +// This will asynchronously run OPA on each file concurrently using goroutines. +func EvalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) (err error) { + _, err = evalE(t, options, jsonFilePaths, resultQuery) + return +} + +// EvalWithOutputE runs `opa eval` on the given JSON files using the configured policy file and result query. Translates to: +// +// opa eval -i $JSONFile -d $RulePath $ResultQuery +// +// This will asynchronously run OPA on each file concurrently using goroutines. +// For each file, the output will be returned on the outputs slice. +func EvalWithOutputE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) (outputs []string, err error) { + return evalE(t, options, jsonFilePaths, resultQuery) +} + +func evalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, resultQuery string) (outputs []string, err error) { downloadedPolicyPath, err := DownloadPolicyE(t, options.RulePath) if err != nil { - return err + return } + outputs = make([]string, len(jsonFilePaths)) wg := new(sync.WaitGroup) wg.Add(len(jsonFilePaths)) errorsOccurred := new(multierror.Error) @@ -74,7 +103,10 @@ func EvalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, res for i, jsonFilePath := range jsonFilePaths { errChan := make(chan error, 1) errChans[i] = errChan - go asyncEval(t, wg, errChan, options, downloadedPolicyPath, jsonFilePath, resultQuery) + + go func(i int, jsonFilePath string) { + outputs[i] = asyncEval(t, wg, errChan, options, downloadedPolicyPath, jsonFilePath, resultQuery) + }(i, jsonFilePath) } wg.Wait() for _, errChan := range errChans { @@ -83,7 +115,7 @@ func EvalE(t testing.TestingT, options *EvalOptions, jsonFilePaths []string, res errorsOccurred = multierror.Append(errorsOccurred, err) } } - return errorsOccurred.ErrorOrNil() + return outputs, errorsOccurred.ErrorOrNil() } // asyncEval is a function designed to be run in a goroutine to asynchronously call `opa eval` on a single input file. @@ -95,7 +127,7 @@ func asyncEval( downloadedPolicyPath string, jsonFilePath string, resultQuery string, -) { +) (output string) { defer wg.Done() cmd := shell.Command{ Command: "opa", @@ -105,7 +137,7 @@ func asyncEval( // opa eval is typically very quick. Logger: logger.Discard, } - err := runCommandWithFullLoggingE(t, options.Logger, cmd) + output, err := runCommandWithFullLoggingE(t, options.Logger, cmd) ruleBasePath := filepath.Base(downloadedPolicyPath) if err == nil { options.Logger.Logf(t, "opa eval passed on file %s (policy %s; query %s)", jsonFilePath, ruleBasePath, resultQuery) @@ -115,10 +147,12 @@ func asyncEval( options.Logger.Logf(t, "DEBUG: rerunning opa eval to query for full data.") cmd.Args = formatOPAEvalArgs(options, downloadedPolicyPath, jsonFilePath, "data") // We deliberately ignore the error here as we want to only return the original error. - runCommandWithFullLoggingE(t, options.Logger, cmd) + output, _ = runCommandWithFullLoggingE(t, options.Logger, cmd) } } errChan <- err + + return } // formatOPAEvalArgs formats the arguments for the `opa eval` command. @@ -146,8 +180,8 @@ func formatOPAEvalArgs(options *EvalOptions, rulePath, jsonFilePath, resultQuery // runCommandWithFullLogging will log the command output in its entirety with buffering. This avoids breaking up the // logs when commands are run concurrently. This is a private function used in the context of opa only because opa runs // very quickly, and the output of opa is hard to parse if it is broken up by interleaved logs. -func runCommandWithFullLoggingE(t testing.TestingT, logger *logger.Logger, cmd shell.Command) error { - output, err := shell.RunCommandAndGetOutputE(t, cmd) +func runCommandWithFullLoggingE(t testing.TestingT, logger *logger.Logger, cmd shell.Command) (output string, err error) { + output, err = shell.RunCommandAndGetOutputE(t, cmd) logger.Logf(t, "Output of command `%s %s`:\n%s", cmd.Command, strings.Join(cmd.Args, " "), output) - return err + return } diff --git a/modules/opa/eval_test.go b/modules/opa/eval_test.go new file mode 100644 index 000000000..2966f9ff7 --- /dev/null +++ b/modules/opa/eval_test.go @@ -0,0 +1,155 @@ +package opa + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEvalWithOutput(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + + policy string + query string + inputs []string + outputs []string + isError bool + }{ + { + name: "Success", + policy: ` + package test + allow { + startswith(input.user, "admin") + } + `, + query: "data.test.allow", + inputs: []string{ + `{"user": "admin-1"}`, + `{"user": "admin-2"}`, + `{"user": "admin-3"}`, + }, + outputs: []string{ + `{ + "result": [{ + "expressions": [{ + "value": true, + "text": "data.test.allow", + "location": { + "row": 1, + "col": 1 + } + }] + }] + }`, + `{ + "result": [{ + "expressions": [{ + "value": true, + "text": "data.test.allow", + "location": { + "row": 1, + "col": 1 + } + }] + }] + }`, + `{ + "result": [{ + "expressions": [{ + "value": true, + "text": "data.test.allow", + "location": { + "row": 1, + "col": 1 + } + }] + }] + }`, + }, + }, + { + name: "ContainsError", + policy: ` + package test + allow { + input.user == "admin" + } + `, + query: "data.test.allow", + isError: true, + inputs: []string{ + `{"user": "admin"}`, + `{"user": "nobody"}`, + }, + outputs: []string{ + `{ + "result": [{ + "expressions": [{ + "value": true, + "text": "data.test.allow", + "location": { + "row": 1, + "col": 1 + } + }] + }] + }`, + `{ + "result": [{ + "expressions": [{ + "value": { + "test": {} + }, + "text": "data", + "location": { + "row": 1, + "col": 1 + } + }] + }] + }`, + }, + }, + } + + createTempFile := func(t *testing.T, name string, content string) string { + f, err := os.CreateTemp(t.TempDir(), name) + require.NoError(t, err) + t.Cleanup(func() { os.Remove(f.Name()) }) + _, err = f.WriteString(content) + require.NoError(t, err) + return f.Name() + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + policy := createTempFile(t, "policy-*.rego", test.policy) + inputs := make([]string, len(test.inputs)) + for i, input := range test.inputs { + f := createTempFile(t, "inputs-*.json", input) + inputs[i] = f + } + + options := &EvalOptions{ + RulePath: policy, + } + + outputs, err := EvalWithOutputE(t, options, inputs, test.query) + if test.isError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + for i, output := range test.outputs { + require.JSONEq(t, output, outputs[i], "output for input: %d", i) + } + }) + } +}