Add support for base and target trees in git ingest, add .tar.gz bund…

…ler (#5181)

* Add support for base and target trees in git ingest, add .tar.gz bundler

* Fix line-too-long lint error

* Rename 'bundle' to 'archive' per feedback from Ed

* Clean up some extra logging, add more details on PR evaluation

* Flag-protect new functionality

* Update new tests for changes to Eval method

* Fix lint /tableflip

cmd/dev/app/rule_type/rttst.go

@@ -197,7 +197,7 @@ func testCmdRun(cmd *cobra.Command, _ []string) error {
 
 	// TODO: use cobra context here
 	ctx := context.Background()
-	eng, err := rtengine.NewRuleTypeEngine(ctx, ruletype, prov, options.WithDataSources(dsRegistry))
+	eng, err := rtengine.NewRuleTypeEngine(ctx, ruletype, prov, nil /*experiments*/, options.WithDataSources(dsRegistry))
 	if err != nil {
 		return fmt.Errorf("cannot create rule type engine: %w", err)
 	}
@@ -326,7 +326,6 @@ func selectAndEval(
 	}
 
 	var evalErr error
-	//var result *interfaces.EvaluationResult
 	if selected {
 		_, evalErr = eng.Eval(ctx, inf.Entity, evalStatus.GetRule().Def, evalStatus.GetRule().Params, evalStatus)
 	} else {

internal/controlplane/handlers_ruletype.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/microcosm-cc/bluemonday"
+	"github.com/open-feature/go-sdk/openfeature"
 	"github.com/yuin/goldmark"
 	"github.com/yuin/goldmark/extension"
 	"github.com/yuin/goldmark/parser"
@@ -23,6 +24,7 @@ import (
 
 	"github.com/mindersec/minder/internal/db"
 	"github.com/mindersec/minder/internal/engine/engcontext"
+	"github.com/mindersec/minder/internal/engine/ingester/git"
 	"github.com/mindersec/minder/internal/flags"
 	"github.com/mindersec/minder/internal/logger"
 	"github.com/mindersec/minder/internal/util"
@@ -175,14 +177,9 @@ func (s *Server) CreateRuleType(
 		return nil, util.UserVisibleError(codes.InvalidArgument, "%s", err)
 	}
 
-	ruleDS := crt.GetRuleType().GetDef().GetEval().GetDataSources()
-	if len(ruleDS) > 0 && !flags.Bool(ctx, s.featureFlags, flags.DataSources) {
-		return nil, util.UserVisibleError(codes.InvalidArgument, "DataSources feature is disabled")
-	}
-
-	prCommentAlert := crt.GetRuleType().GetDef().GetAlert().GetPullRequestComment()
-	if prCommentAlert != nil && !flags.Bool(ctx, s.featureFlags, flags.PRCommentAlert) {
-		return nil, util.UserVisibleError(codes.InvalidArgument, "Pull request comment alert type is disabled")
+	ruleDef := crt.GetRuleType().GetDef()
+	if err := checkRuleDefinitionFlags(ctx, s.featureFlags, ruleDef); err != nil {
+		return nil, err
 	}
 
 	newRuleType, err := db.WithTransaction(s.store, func(qtx db.ExtendQuerier) (*minderv1.RuleType, error) {
@@ -204,6 +201,27 @@ func (s *Server) CreateRuleType(
 	}, nil
 }
 
+func checkRuleDefinitionFlags(
+	ctx context.Context, featureFlags openfeature.IClient, ruleDef *minderv1.RuleType_Definition) *util.NiceStatus {
+	ruleDS := ruleDef.GetEval().GetDataSources()
+	if len(ruleDS) > 0 && !flags.Bool(ctx, featureFlags, flags.DataSources) {
+		return util.UserVisibleError(codes.InvalidArgument, "DataSources feature is disabled")
+	}
+
+	prCommentAlert := ruleDef.GetAlert().GetPullRequestComment()
+	if prCommentAlert != nil && !flags.Bool(ctx, featureFlags, flags.PRCommentAlert) {
+		return util.UserVisibleError(codes.InvalidArgument, "Pull request comment alert type is disabled")
+	}
+
+	usesGitPR := ruleDef.GetIngest().GetType() == git.GitRuleDataIngestType &&
+		ruleDef.GetInEntity() == minderv1.PullRequestEntity.String()
+	if usesGitPR && !flags.Bool(ctx, featureFlags, flags.GitPRDiffs) {
+		return util.UserVisibleError(codes.InvalidArgument, "Git pull request ingest is disabled")
+	}
+
+	return nil
+}
+
 // UpdateRuleType is a method to update a rule type
 func (s *Server) UpdateRuleType(
 	ctx context.Context,
@@ -227,9 +245,9 @@ func (s *Server) UpdateRuleType(
 		return nil, util.UserVisibleError(codes.InvalidArgument, "%s", err)
 	}
 
-	ruleDS := urt.GetRuleType().GetDef().GetEval().GetDataSources()
-	if len(ruleDS) > 0 && !flags.Bool(ctx, s.featureFlags, flags.DataSources) {
-		return nil, util.UserVisibleError(codes.InvalidArgument, "DataSources feature is disabled")
+	ruleDef := urt.GetRuleType().GetDef()
+	if err := checkRuleDefinitionFlags(ctx, s.featureFlags, ruleDef); err != nil {
+		return nil, err
 	}
 
 	updatedRuleType, err := db.WithTransaction(s.store, func(qtx db.ExtendQuerier) (*minderv1.RuleType, error) {

internal/engine/eval/eval.go

@@ -10,6 +10,8 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/open-feature/go-sdk/openfeature"
+
 	"github.com/mindersec/minder/internal/engine/eval/homoglyphs/application"
 	"github.com/mindersec/minder/internal/engine/eval/jq"
 	"github.com/mindersec/minder/internal/engine/eval/rego"
@@ -26,6 +28,7 @@ func NewRuleEvaluator(
 	ctx context.Context,
 	ruletype *minderv1.RuleType,
 	provider provinfv1.Provider,
+	featureFlags openfeature.IClient,
 	opts ...eoptions.Option,
 ) (interfaces.Evaluator, error) {
 	e := ruletype.Def.GetEval()
@@ -41,7 +44,7 @@ func NewRuleEvaluator(
 		}
 		return jq.NewJQEvaluator(e.GetJq(), opts...)
 	case rego.RegoEvalType:
-		return rego.NewRegoEvaluator(e.GetRego(), opts...)
+		return rego.NewRegoEvaluator(e.GetRego(), featureFlags, opts...)
 	case vulncheck.VulncheckEvalType:
 		client, err := provinfv1.As[provinfv1.GitHub](provider)
 		if err != nil {

internal/engine/eval/eval_test.go

@@ -71,7 +71,7 @@ func TestNewRuleEvaluatorWorks(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			got, err := eval.NewRuleEvaluator(context.Background(), tt.args.rt, nil)
+			got, err := eval.NewRuleEvaluator(context.Background(), tt.args.rt, nil, nil)
 			assert.NoError(t, err, "unexpected error")
 			assert.NotNil(t, got, "unexpected nil")
 		})
@@ -146,7 +146,7 @@ func TestNewRuleEvaluatorFails(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			got, err := eval.NewRuleEvaluator(context.Background(), tt.args.rt, nil)
+			got, err := eval.NewRuleEvaluator(context.Background(), tt.args.rt, nil, nil)
 			assert.Error(t, err, "should have errored")
 			assert.Nil(t, got, "should be nil")
 		})

internal/engine/eval/rego/eval.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/open-feature/go-sdk/openfeature"
 	"github.com/open-policy-agent/opa/rego"
 	"github.com/open-policy-agent/opa/topdown/print"
 	"google.golang.org/protobuf/reflect/protoreflect"
@@ -38,10 +39,11 @@ const (
 // It initializes the rego engine and evaluates the rules
 // The default rego package is "minder"
 type Evaluator struct {
-	cfg         *Config
-	regoOpts    []func(*rego.Rego)
-	reseval     resultEvaluator
-	datasources *v1datasources.DataSourceRegistry
+	cfg          *Config
+	featureFlags openfeature.IClient
+	regoOpts     []func(*rego.Rego)
+	reseval      resultEvaluator
+	datasources  *v1datasources.DataSourceRegistry
 }
 
 // Input is the input for the rego evaluator
@@ -70,6 +72,7 @@ var _ print.Hook = (*hook)(nil)
 // NewRegoEvaluator creates a new rego evaluator
 func NewRegoEvaluator(
 	cfg *minderv1.RuleType_Definition_Eval_Rego,
+	featureFlags openfeature.IClient,
 	opts ...eoptions.Option,
 ) (*Evaluator, error) {
 	c, err := parseConfig(cfg)
@@ -80,8 +83,9 @@ func NewRegoEvaluator(
 	re := c.getEvalType()
 
 	eval := &Evaluator{
-		cfg:     c,
-		reseval: re,
+		cfg:          c,
+		featureFlags: featureFlags,
+		reseval:      re,
 		regoOpts: []func(*rego.Rego){
 			re.getQuery(),
 			rego.Module(MinderRegoFile, c.Def),
@@ -123,7 +127,7 @@ func (e *Evaluator) Eval(
 	regoFuncOptions := []func(*rego.Rego){}
 
 	// Initialize the built-in minder library rego functions
-	regoFuncOptions = append(regoFuncOptions, instantiateRegoLib(res)...)
+	regoFuncOptions = append(regoFuncOptions, instantiateRegoLib(ctx, e.featureFlags, res)...)
 
 	// If the evaluator has data sources defined, expose their functions
 	regoFuncOptions = append(regoFuncOptions, buildDataSourceOptions(res, e.datasources)...)

internal/engine/eval/rego/fuzz_test.go

@@ -20,6 +20,7 @@ func FuzzRegoEval(f *testing.F) {
 				Type: ConstraintsEvaluationType.String(),
 				Def:  policy,
 			},
+			nil,
 		)
 		if err != nil {
 			return