mindev: Add ability to parse and use data sources (#5055)

This allows us to easily test data sources with `mindev`

Signed-off-by: Juan Antonio Osorio <[email protected]>

cmd/dev/app/rule_type/rttst.go

@@ -17,12 +17,14 @@ import (
 	"github.com/spf13/viper"
 	"google.golang.org/protobuf/reflect/protoreflect"
 
+	internalds "github.com/mindersec/minder/internal/datasources"
 	"github.com/mindersec/minder/internal/db"
 	"github.com/mindersec/minder/internal/engine/actions"
 	"github.com/mindersec/minder/internal/engine/entities"
 	"github.com/mindersec/minder/internal/engine/errors"
 	"github.com/mindersec/minder/internal/engine/eval/rego"
 	engif "github.com/mindersec/minder/internal/engine/interfaces"
+	"github.com/mindersec/minder/internal/engine/options"
 	entModels "github.com/mindersec/minder/internal/entities/models"
 	entProps "github.com/mindersec/minder/internal/entities/properties"
 	"github.com/mindersec/minder/internal/providers/credentials"
@@ -36,6 +38,7 @@ import (
 	"github.com/mindersec/minder/internal/util/jsonyaml"
 	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
 	serverconfig "github.com/mindersec/minder/pkg/config/server"
+	v1datasources "github.com/mindersec/minder/pkg/datasources/v1"
 	"github.com/mindersec/minder/pkg/engine/selectors"
 	"github.com/mindersec/minder/pkg/engine/v1/rtengine"
 	"github.com/mindersec/minder/pkg/profiles"
@@ -63,6 +66,7 @@ func CmdTest() *cobra.Command {
 	testCmd.Flags().StringP("remediate-metadata", "", "", "YAML file containing the remediate metadata (optional)")
 	testCmd.Flags().StringP("token", "t", "", "token to authenticate to the provider."+
 		"Can also be set via the TEST_AUTH_TOKEN environment variable.")
+	testCmd.Flags().StringArrayP("data-source", "d", []string{}, "YAML file containing the data source to test the rule with")
 
 	if err := testCmd.MarkFlagRequired("rule-type"); err != nil {
 		fmt.Fprintf(os.Stderr, "Error marking flag as required: %s\n", err)
@@ -84,6 +88,7 @@ func CmdTest() *cobra.Command {
 	return testCmd
 }
 
+//nolint:gocyclo // this function is a cobra command and is expected to be complex
 func testCmdRun(cmd *cobra.Command, _ []string) error {
 	rtpath := cmd.Flag("rule-type")
 	epath := cmd.Flag("entity")
@@ -94,6 +99,24 @@ func testCmdRun(cmd *cobra.Command, _ []string) error {
 	providerclass := cmd.Flag("provider")
 	providerconfig := cmd.Flag("provider-config")
 
+	dataSourceFileStrings, err := cmd.Flags().GetStringArray("data-source")
+	if err != nil {
+		return fmt.Errorf("error getting data source files: %w", err)
+	}
+
+	dataSourcefiles, err := getDataSourceFiles(dataSourceFileStrings)
+	if err != nil {
+		return fmt.Errorf("error getting data source files: %w", err)
+	}
+
+	// close the files when done
+	defer func() {
+		for _, f := range dataSourcefiles {
+			//nolint:gosec // we are closing the file
+			f.Close()
+		}
+	}()
+
 	// set rego env variable for debugging
 	if err := os.Setenv(rego.EnablePrintEnvVar, "true"); err != nil {
 		cmd.Printf("Unable to set %s environment variable: %s\n", rego.EnablePrintEnvVar, err)
@@ -167,9 +190,14 @@ func testCmdRun(cmd *cobra.Command, _ []string) error {
 		Alert:     actionOptFromString(profile.Alert, models.ActionOptOff),
 	}
 
+	dsRegistry, err := getDataSources(dataSourcefiles)
+	if err != nil {
+		return fmt.Errorf("error getting data sources: %w", err)
+	}
+
 	// TODO: use cobra context here
 	ctx := context.Background()
-	eng, err := rtengine.NewRuleTypeEngine(ctx, ruletype, prov)
+	eng, err := rtengine.NewRuleTypeEngine(ctx, ruletype, prov, options.WithDataSources(dsRegistry))
 	if err != nil {
 		return fmt.Errorf("cannot create rule type engine: %w", err)
 	}
@@ -475,3 +503,39 @@ func actionOptFromString(s *string, defAction models.ActionOpt) models.ActionOpt
 
 	return models.ActionOptUnknown
 }
+
+func getDataSources(readers []*os.File) (*v1datasources.DataSourceRegistry, error) {
+	reg := v1datasources.NewDataSourceRegistry()
+	for _, r := range readers {
+		fname := r.Name()
+		ds := &minderv1.DataSource{}
+		if err := minderv1.ParseResourceProto(r, ds); err != nil {
+			return nil, fmt.Errorf("error parsing data source %s: %w", fname, err)
+		}
+
+		// TODO: Add data source validation here.
+
+		intds, err := internalds.BuildFromProtobuf(ds)
+		if err != nil {
+			return nil, fmt.Errorf("error building data source %s: %w", fname, err)
+		}
+
+		if err := reg.RegisterDataSource(ds.GetName(), intds); err != nil {
+			return nil, fmt.Errorf("error registering data source %s: %w", fname, err)
+		}
+	}
+
+	return reg, nil
+}
+
+func getDataSourceFiles(files []string) ([]*os.File, error) {
+	dataSourceFiles := make([]*os.File, 0, len(files))
+	for _, f := range files {
+		file, err := os.Open(filepath.Clean(f))
+		if err != nil {
+			return nil, fmt.Errorf("error opening file: %w", err)
+		}
+		dataSourceFiles = append(dataSourceFiles, file)
+	}
+	return dataSourceFiles, nil
+}