diff --git a/internal/engine/eval/rego/datasources.go b/internal/engine/eval/rego/datasources.go new file mode 100644 index 0000000000..355dc99722 --- /dev/null +++ b/internal/engine/eval/rego/datasources.go @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors +// SPDX-License-Identifier: Apache-2.0 + +package rego + +import ( + "fmt" + "strings" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/rego" + "github.com/open-policy-agent/opa/types" + + v1datasources "github.com/mindersec/minder/pkg/datasources/v1" +) + +// RegisterDataSources implements the Eval interface. +func (e *Evaluator) RegisterDataSources(dsr *v1datasources.DataSourceRegistry) { + for key, dsf := range dsr.GetFuncs() { + fmt.Printf("Registering data source %s\n", key) + e.regoOpts = append(e.regoOpts, buildFromDataSource(key, dsf)) + } +} + +// buildFromDataSource builds a rego function from a data source function. +// It takes a DataSourceFuncDef and returns a function that can be used to +// register the function with the rego engine. +func buildFromDataSource(key v1datasources.DataSourceFuncKey, dsf v1datasources.DataSourceFuncDef) func(*rego.Rego) { + k := normalizeKey(key) + return rego.Function1( + ®o.Function{ + Name: k, + Decl: types.NewFunction(types.Args(types.A), types.A), + }, + func(_ rego.BuiltinContext, obj *ast.Term) (*ast.Term, error) { + // Convert the AST value back to a Go interface{} + jsonObj, err := ast.JSON(obj.Value) + if err != nil { + return nil, err + } + + if err := dsf.ValidateArgs(obj); err != nil { + return nil, err + } + + // Call the data source function + ret, err := dsf.Call(jsonObj) + if err != nil { + return nil, err + } + + val, err := ast.InterfaceToValue(ret) + if err != nil { + return nil, err + } + + return ast.NewTerm(val), nil + }, + ) +} + +// This converts the data source function key into a format that can be used in the rego query. +// For example, if the key is "aws.ec2.instances", it will +// be converted to "minder.data.aws.ec2.instances". +// It also normalizes the key to lowercase (which should have already been done) +// and converts any "-" to "_", finally it removes any special characters. +func normalizeKey(key v1datasources.DataSourceFuncKey) string { + low := strings.ToLower(key.String()) + underscore := strings.ReplaceAll(low, "-", "_") + // Remove any special characters + norm := strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '_' || r == '.' { + return r + } + return -1 + }, underscore) + return fmt.Sprintf("minder.datasource.%s", norm) +} diff --git a/internal/engine/eval/rego/rego_test.go b/internal/engine/eval/rego/rego_test.go index 1e8477d86c..b5834f984b 100644 --- a/internal/engine/eval/rego/rego_test.go +++ b/internal/engine/eval/rego/rego_test.go @@ -10,10 +10,14 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" engerrors "github.com/mindersec/minder/internal/engine/errors" "github.com/mindersec/minder/internal/engine/eval/rego" + "github.com/mindersec/minder/internal/engine/options" minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1" + v1datasources "github.com/mindersec/minder/pkg/datasources/v1" + v1mockds "github.com/mindersec/minder/pkg/datasources/v1/mock" "github.com/mindersec/minder/pkg/engine/v1/interfaces" ) @@ -454,3 +458,59 @@ violations[{"msg": msg}] { &interfaces.Result{Object: map[string]any{}}) assert.Error(t, err, "should have failed to evaluate") } + +func TestCustomDatasourceRegister(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + + fds := v1mockds.NewMockDataSource(ctrl) + fdsf := v1mockds.NewMockDataSourceFuncDef(ctrl) + + fds.EXPECT().GetFuncs().Return(map[v1datasources.DataSourceFuncKey]v1datasources.DataSourceFuncDef{ + "source": fdsf, + }).AnyTimes() + + fdsf.EXPECT().ValidateArgs(gomock.Any()).Return(nil).AnyTimes() + + fdsr := v1datasources.NewDataSourceRegistry() + + err := fdsr.RegisterDataSource("fake", fds) + require.NoError(t, err, "could not register data source") + + e, err := rego.NewRegoEvaluator( + &minderv1.RuleType_Definition_Eval_Rego{ + Type: rego.DenyByDefaultEvaluationType.String(), + Def: ` +package minder + +default allow = false + +allow { + minder.datasource.fake.source({"datasourcetest": input.ingested.data}) == "foo" +}`, + }, + options.WithDataSources(fdsr), + ) + require.NoError(t, err, "could not create evaluator") + + emptyPol := map[string]any{} + + // Matches + fdsf.EXPECT().Call(gomock.Any()).Return("foo", nil) + err = e.Eval(context.Background(), emptyPol, nil, &interfaces.Result{ + Object: map[string]any{ + "data": "foo", + }, + }) + require.NoError(t, err, "could not evaluate") + + // Doesn't match + fdsf.EXPECT().Call(gomock.Any()).Return("bar", nil) + err = e.Eval(context.Background(), emptyPol, nil, &interfaces.Result{ + Object: map[string]any{ + "data": "bar", + }, + }) + require.ErrorIs(t, err, engerrors.ErrEvaluationFailed, "should have failed the evaluation") +} diff --git a/internal/engine/options/options.go b/internal/engine/options/options.go index 7e2ceb1353..0da6223418 100644 --- a/internal/engine/options/options.go +++ b/internal/engine/options/options.go @@ -8,6 +8,7 @@ package options import ( "github.com/open-feature/go-sdk/openfeature" + v1datasources "github.com/mindersec/minder/pkg/datasources/v1" "github.com/mindersec/minder/pkg/engine/v1/interfaces" ) @@ -33,3 +34,23 @@ func WithFlagsClient(client openfeature.IClient) Option { return inner.SetFlagsClient(client) } } + +// SupportsDataSources interface advertises the fact that the implementer +// can register data sources with the evaluator. +type SupportsDataSources interface { + RegisterDataSources(ds *v1datasources.DataSourceRegistry) +} + +// WithDataSources provides the evaluation engine with a list of data sources +// to register. In case the given evaluator does not support data sources, +// WithDataSources silently ignores the error. +func WithDataSources(ds *v1datasources.DataSourceRegistry) Option { + return func(e interfaces.Evaluator) error { + inner, ok := e.(SupportsDataSources) + if !ok { + return nil + } + inner.RegisterDataSources(ds) + return nil + } +} diff --git a/pkg/datasources/v1/datasources.go b/pkg/datasources/v1/datasources.go new file mode 100644 index 0000000000..2c68082b2d --- /dev/null +++ b/pkg/datasources/v1/datasources.go @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright 2024 The Minder Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package v1 provides the interfaces and types for the data sources. +package v1 + +//go:generate go run go.uber.org/mock/mockgen -package mock_$GOPACKAGE -destination=./mock/$GOFILE -source=./$GOFILE + +// DataSourceFuncKey is the key that uniquely identifies a data source function. +type DataSourceFuncKey string + +// String returns the string representation of the data source function key. +func (k DataSourceFuncKey) String() string { + return string(k) +} + +// DataSourceFuncDef is the definition of a data source function. +// It contains the key that uniquely identifies the function and the arguments +// that the function can take. +type DataSourceFuncDef interface { + // ValidateArgs validates the arguments of the function. + ValidateArgs(obj any) error + // ValidateUpdate validates the update to the data source. + // The data source implementation should respect the update and return an error + // if the update is invalid. + ValidateUpdate(obj any) error + // Call calls the function with the given arguments. + // It is the responsibility of the data source implementation to handle the call. + // It is also the responsibility of the caller to validate the arguments + // before calling the function. + Call(args any) (any, error) +} + +// DataSource is the interface that a data source must implement. +// It implements several functions that will be used by the engine to +// interact with external systems. These get taken into used by the Evaluator. +// Moreover, a data source must be able to validate an update to itself. +type DataSource interface { + // Returns the registered name of the data source. + GetName() string + + // GetFuncs returns the functions that the data source provides. + GetFuncs() map[DataSourceFuncKey]DataSourceFuncDef +} diff --git a/pkg/datasources/v1/mock/datasources.go b/pkg/datasources/v1/mock/datasources.go new file mode 100644 index 0000000000..9b3fa514d7 --- /dev/null +++ b/pkg/datasources/v1/mock/datasources.go @@ -0,0 +1,136 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./datasources.go +// +// Generated by this command: +// +// mockgen -package mock_v1 -destination=./mock/datasources.go -source=./datasources.go +// + +// Package mock_v1 is a generated GoMock package. +package mock_v1 + +import ( + reflect "reflect" + + v1 "github.com/mindersec/minder/pkg/datasources/v1" + gomock "go.uber.org/mock/gomock" +) + +// MockDataSourceFuncDef is a mock of DataSourceFuncDef interface. +type MockDataSourceFuncDef struct { + ctrl *gomock.Controller + recorder *MockDataSourceFuncDefMockRecorder + isgomock struct{} +} + +// MockDataSourceFuncDefMockRecorder is the mock recorder for MockDataSourceFuncDef. +type MockDataSourceFuncDefMockRecorder struct { + mock *MockDataSourceFuncDef +} + +// NewMockDataSourceFuncDef creates a new mock instance. +func NewMockDataSourceFuncDef(ctrl *gomock.Controller) *MockDataSourceFuncDef { + mock := &MockDataSourceFuncDef{ctrl: ctrl} + mock.recorder = &MockDataSourceFuncDefMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataSourceFuncDef) EXPECT() *MockDataSourceFuncDefMockRecorder { + return m.recorder +} + +// Call mocks base method. +func (m *MockDataSourceFuncDef) Call(args any) (any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Call", args) + ret0, _ := ret[0].(any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Call indicates an expected call of Call. +func (mr *MockDataSourceFuncDefMockRecorder) Call(args any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Call", reflect.TypeOf((*MockDataSourceFuncDef)(nil).Call), args) +} + +// ValidateArgs mocks base method. +func (m *MockDataSourceFuncDef) ValidateArgs(obj any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateArgs", obj) + ret0, _ := ret[0].(error) + return ret0 +} + +// ValidateArgs indicates an expected call of ValidateArgs. +func (mr *MockDataSourceFuncDefMockRecorder) ValidateArgs(obj any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateArgs", reflect.TypeOf((*MockDataSourceFuncDef)(nil).ValidateArgs), obj) +} + +// ValidateUpdate mocks base method. +func (m *MockDataSourceFuncDef) ValidateUpdate(obj any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateUpdate", obj) + ret0, _ := ret[0].(error) + return ret0 +} + +// ValidateUpdate indicates an expected call of ValidateUpdate. +func (mr *MockDataSourceFuncDefMockRecorder) ValidateUpdate(obj any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateUpdate", reflect.TypeOf((*MockDataSourceFuncDef)(nil).ValidateUpdate), obj) +} + +// MockDataSource is a mock of DataSource interface. +type MockDataSource struct { + ctrl *gomock.Controller + recorder *MockDataSourceMockRecorder + isgomock struct{} +} + +// MockDataSourceMockRecorder is the mock recorder for MockDataSource. +type MockDataSourceMockRecorder struct { + mock *MockDataSource +} + +// NewMockDataSource creates a new mock instance. +func NewMockDataSource(ctrl *gomock.Controller) *MockDataSource { + mock := &MockDataSource{ctrl: ctrl} + mock.recorder = &MockDataSourceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataSource) EXPECT() *MockDataSourceMockRecorder { + return m.recorder +} + +// GetFuncs mocks base method. +func (m *MockDataSource) GetFuncs() map[v1.DataSourceFuncKey]v1.DataSourceFuncDef { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFuncs") + ret0, _ := ret[0].(map[v1.DataSourceFuncKey]v1.DataSourceFuncDef) + return ret0 +} + +// GetFuncs indicates an expected call of GetFuncs. +func (mr *MockDataSourceMockRecorder) GetFuncs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFuncs", reflect.TypeOf((*MockDataSource)(nil).GetFuncs)) +} + +// GetName mocks base method. +func (m *MockDataSource) GetName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetName") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetName indicates an expected call of GetName. +func (mr *MockDataSourceMockRecorder) GetName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetName", reflect.TypeOf((*MockDataSource)(nil).GetName)) +} diff --git a/pkg/datasources/v1/registry.go b/pkg/datasources/v1/registry.go new file mode 100644 index 0000000000..a1b71a92f5 --- /dev/null +++ b/pkg/datasources/v1/registry.go @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright 2024 The Minder Authors +// SPDX-License-Identifier: Apache-2.0 + +package v1 + +import ( + "errors" + "fmt" + + "github.com/puzpuzpuz/xsync/v3" +) + +var ( + // ErrDuplicateDataSourceFuncKey is the error returned when a data source + // function key is already registered. + ErrDuplicateDataSourceFuncKey = errors.New("duplicate data source function key") +) + +// DataSourceRegistry is the interface that a data source registry must implement. +// It provides methods to register a data source and get all functions that +// data sources provide globally. +type DataSourceRegistry struct { + r *xsync.MapOf[DataSourceFuncKey, DataSourceFuncDef] +} + +// NewDataSourceRegistry creates a new data source registry. +func NewDataSourceRegistry() *DataSourceRegistry { + return &DataSourceRegistry{ + r: xsync.NewMapOf[DataSourceFuncKey, DataSourceFuncDef](), + } +} + +// RegisterDataSource registers a data source with the registry. +// Note that the name of the data source must be unique. +func (reg *DataSourceRegistry) RegisterDataSource(name string, ds DataSource) (err error) { + for key, f := range ds.GetFuncs() { + funckey := makeKey(name, key) + if _, ok := reg.r.Load(funckey); ok { + return fmt.Errorf("%w: %s", ErrDuplicateDataSourceFuncKey, funckey) + } + + // We only flush the store if there was no error + defer func() { + if err == nil { + reg.r.Store(funckey, f) + } + }() + } + + return nil +} + +// GetFuncs returns all functions that data sources provide globally. +func (reg *DataSourceRegistry) GetFuncs() map[DataSourceFuncKey]DataSourceFuncDef { + out := make(map[DataSourceFuncKey]DataSourceFuncDef, reg.r.Size()) + reg.r.Range(func(key DataSourceFuncKey, value DataSourceFuncDef) bool { + out[key] = value + return true + }) + + return out +} + +func makeKey(name string, key DataSourceFuncKey) DataSourceFuncKey { + return DataSourceFuncKey(name + "." + key.String()) +}