diff --git a/apps/cnspec/cmd/scan.go b/apps/cnspec/cmd/scan.go index b19ffc6f7..af4edf900 100644 --- a/apps/cnspec/cmd/scan.go +++ b/apps/cnspec/cmd/scan.go @@ -19,6 +19,7 @@ import ( "go.mondoo.com/cnquery/v9/cli/inventoryloader" "go.mondoo.com/cnquery/v9/cli/theme" "go.mondoo.com/cnquery/v9/logger" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/providers" "go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory" "go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin" @@ -347,9 +348,10 @@ func (c *scanConfig) loadPolicies(ctx context.Context) error { } bundle.ConvertQuerypacks() + conf := mqlc.NewConfig(c.runtime.Schema(), cnquery.DefaultFeatures) _, err = bundle.CompileExt(ctx, policy.BundleCompileConf{ - Schema: c.runtime.Schema(), + CompilerConfig: conf, // We don't care about failing queries for local runs. We may only // process a subset of all the queries in the bundle. When we receive // things from the server, upstream can filter things for us. But running diff --git a/internal/datalakes/inmemory/policyhub.go b/internal/datalakes/inmemory/policyhub.go index e7de7913c..364de7670 100644 --- a/internal/datalakes/inmemory/policyhub.go +++ b/internal/datalakes/inmemory/policyhub.go @@ -538,7 +538,7 @@ func (db *Db) fixInvalidatedPolicy(ctx context.Context, wrap *wrapPolicy) error func(ctx context.Context, mrn string) (*policy.Policy, error) { return db.GetValidatedPolicy(ctx, mrn) }, func(ctx context.Context, mrn string) (*explorer.Mquery, error) { return db.GetQuery(ctx, mrn) }, nil, - db.services.Schema(), + db.services.NewCompilerConfig(), ) ok := db.cache.Set(dbIDPolicy+wrap.Policy.Mrn, *wrap, 2) diff --git a/internal/datalakes/inmemory/policyresolver.go b/internal/datalakes/inmemory/policyresolver.go index 958672d92..83c5fbcfb 100644 --- a/internal/datalakes/inmemory/policyresolver.go +++ b/internal/datalakes/inmemory/policyresolver.go @@ -188,7 +188,7 @@ func (db *Db) mutatePolicy(ctx context.Context, mrn string, actions map[string]e func(ctx context.Context, mrn string) (*policy.Policy, error) { return db.GetValidatedPolicy(ctx, mrn) }, func(ctx context.Context, mrn string) (*explorer.Mquery, error) { return db.GetQuery(ctx, mrn) }, nil, - db.services.Schema(), + db.services.NewCompilerConfig(), ) if err != nil { return policyw, true, err @@ -350,6 +350,8 @@ func (db *Db) refreshDependentAssetFilters(ctx context.Context, startPolicy wrap } for len(needsUpdate) > 0 { + conf := db.services.NewCompilerConfig() + for k, policyw := range needsUpdate { err := db.refreshAssetFilters(ctx, &policyw) if err != nil { @@ -361,7 +363,7 @@ func (db *Db) refreshDependentAssetFilters(ctx context.Context, startPolicy wrap func(ctx context.Context, mrn string) (*policy.Policy, error) { return db.GetValidatedPolicy(ctx, mrn) }, func(ctx context.Context, mrn string) (*explorer.Mquery, error) { return db.GetQuery(ctx, mrn) }, nil, - db.services.Schema(), + conf, ) if err != nil { return err @@ -861,7 +863,7 @@ func (db *Db) SetProps(ctx context.Context, req *explorer.PropsReq) error { func(ctx context.Context, mrn string) (*policy.Policy, error) { return db.GetValidatedPolicy(ctx, mrn) }, func(ctx context.Context, mrn string) (*explorer.Mquery, error) { return db.GetQuery(ctx, mrn) }, nil, - db.services.Schema(), + db.services.NewCompilerConfig(), ) if err != nil { return err diff --git a/policy/bundle.go b/policy/bundle.go index b5476feb6..52ec44649 100644 --- a/policy/bundle.go +++ b/policy/bundle.go @@ -13,10 +13,12 @@ import ( "github.com/pkg/errors" "github.com/rs/zerolog/log" + "go.mondoo.com/cnquery/v9" "go.mondoo.com/cnquery/v9/checksums" "go.mondoo.com/cnquery/v9/explorer" "go.mondoo.com/cnquery/v9/llx" "go.mondoo.com/cnquery/v9/logger" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/mrn" "go.mondoo.com/cnquery/v9/utils/multierr" "sigs.k8s.io/yaml" @@ -712,13 +714,13 @@ func topologicalSortQueriesDFS(queryMrn string, queriesMap map[string]*explorer. // Compile a bundle. See CompileExt for a full description. func (p *Bundle) Compile(ctx context.Context, schema llx.Schema, library Library) (*PolicyBundleMap, error) { return p.CompileExt(ctx, BundleCompileConf{ - Schema: schema, - Library: library, + CompilerConfig: mqlc.NewConfig(schema, cnquery.DefaultFeatures), + Library: library, }) } type BundleCompileConf struct { - Schema llx.Schema + mqlc.CompilerConfig Library Library RemoveFailing bool } @@ -811,7 +813,7 @@ func (p *Bundle) CompileExt(ctx context.Context, conf BundleCompileConf) (*Polic if policy.ComputedFilters == nil || policy.ComputedFilters.Items == nil { policy.ComputedFilters = &explorer.Filters{Items: map[string]*explorer.Mquery{}} } - if err = policy.ComputedFilters.Compile(ownerMrn, cache.conf.Schema); err != nil { + if err = policy.ComputedFilters.Compile(ownerMrn, conf.CompilerConfig); err != nil { return nil, multierr.Wrap(err, "failed to compile policy filters") } @@ -820,7 +822,7 @@ func (p *Bundle) CompileExt(ctx context.Context, conf BundleCompileConf) (*Polic group := policy.Groups[i] // When filters are initially added they haven't been compiled - if err = group.Filters.Compile(ownerMrn, cache.conf.Schema); err != nil { + if err = group.Filters.Compile(ownerMrn, conf.CompilerConfig); err != nil { return nil, multierr.Wrap(err, "failed to compile policy group filters") } @@ -891,7 +893,7 @@ func (p *Bundle) CompileExt(ctx context.Context, conf BundleCompileConf) (*Polic return nil, errors.New("failed to validate policy: " + err.Error()) } - err = bundleMap.ValidatePolicy(ctx, policy, cache.conf.Schema) + err = bundleMap.ValidatePolicy(ctx, policy, cache.conf.CompilerConfig) if err != nil { return nil, errors.New("failed to validate policy: " + err.Error()) } @@ -1066,7 +1068,7 @@ func (c *bundleCache) precompileQuery(query *explorer.Mquery, policy *Policy) *e } // filters have no dependencies, so we can compile them early - if err := query.Filters.Compile(c.ownerMrn, c.conf.Schema); err != nil { + if err := query.Filters.Compile(c.ownerMrn, c.conf.CompilerConfig); err != nil { c.errors = append(c.errors, errors.New("failed to compile filters for query "+query.Mrn)) return nil } @@ -1100,7 +1102,7 @@ func (c *bundleCache) precompileQuery(query *explorer.Mquery, policy *Policy) *e // dependencies have been processed. Properties must be compiled. Connected // queries may not be ready yet, but we have to have precompiled them. func (c *bundleCache) compileQuery(query *explorer.Mquery) { - _, err := query.RefreshChecksumAndType(c.lookupQuery, c.lookupProp, c.conf.Schema) + _, err := query.RefreshChecksumAndType(c.lookupQuery, c.lookupProp, c.conf.CompilerConfig) if err != nil { if c.conf.RemoveFailing { c.removeQueries[query.Mrn] = struct{}{} @@ -1133,7 +1135,7 @@ func (c *bundleCache) compileProp(prop *explorer.Property) error { name = m.Basename() } - if _, err := prop.RefreshChecksumAndType(c.conf.Schema); err != nil { + if _, err := prop.RefreshChecksumAndType(c.conf.CompilerConfig); err != nil { return err } diff --git a/policy/bundle_map.go b/policy/bundle_map.go index d39277a9c..6618a16da 100644 --- a/policy/bundle_map.go +++ b/policy/bundle_map.go @@ -11,6 +11,7 @@ import ( "github.com/pkg/errors" "go.mondoo.com/cnquery/v9/explorer" "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/mrn" "go.mondoo.com/cnquery/v9/utils/sortx" ) @@ -173,13 +174,13 @@ func sortPolicies(p *Policy, bundle *PolicyBundleMap, indexer map[string]struct{ } // ValidatePolicy against the given bundle -func (p *PolicyBundleMap) ValidatePolicy(ctx context.Context, policy *Policy, schema llx.Schema) error { +func (p *PolicyBundleMap) ValidatePolicy(ctx context.Context, policy *Policy, conf mqlc.CompilerConfig) error { if !mrn.IsValid(policy.Mrn) { return errors.New("policy MRN is not valid: " + policy.Mrn) } for i := range policy.Groups { - if err := p.validateGroup(ctx, policy.Groups[i], policy.Mrn, schema); err != nil { + if err := p.validateGroup(ctx, policy.Groups[i], policy.Mrn, conf); err != nil { return err } } @@ -195,7 +196,7 @@ func (p *PolicyBundleMap) ValidatePolicy(ctx context.Context, policy *Policy, sc return nil } -func (p *PolicyBundleMap) validateGroup(ctx context.Context, group *PolicyGroup, policyMrn string, schema llx.Schema) error { +func (p *PolicyBundleMap) validateGroup(ctx context.Context, group *PolicyGroup, policyMrn string, conf mqlc.CompilerConfig) error { if group == nil { return errors.New("spec cannot be nil") } @@ -204,7 +205,7 @@ func (p *PolicyBundleMap) validateGroup(ctx context.Context, group *PolicyGroup, // since asset filters are run beforehand and don't make it into the report // we don't store their code bundles separately for _, query := range group.Filters.Items { - _, err := query.RefreshAsFilter(policyMrn, schema) + _, err := query.RefreshAsFilter(policyMrn, conf) if err != nil { return err } diff --git a/policy/bundle_test.go b/policy/bundle_test.go index ce8504040..512c35b55 100644 --- a/policy/bundle_test.go +++ b/policy/bundle_test.go @@ -17,7 +17,6 @@ import ( "github.com/stretchr/testify/require" "go.mondoo.com/cnquery/v9/explorer" "go.mondoo.com/cnquery/v9/providers" - "go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils" "go.mondoo.com/cnspec/v9/internal/datalakes/inmemory" "go.mondoo.com/cnspec/v9/policy" ) @@ -232,7 +231,7 @@ func TestBundleCompile(t *testing.T) { require.NoError(t, err) require.NotNil(t, bundle) - bundlemap, err := bundle.Compile(context.Background(), schema, nil) + bundlemap, err := bundle.Compile(context.Background(), conf.Schema, nil) require.NoError(t, err) require.NotNil(t, bundlemap) @@ -357,13 +356,13 @@ func TestStableMqueryChecksum(t *testing.T) { require.NoError(t, err) require.NotNil(t, bundle) - bundlemap, err := bundle.Compile(context.Background(), schema, nil) + bundlemap, err := bundle.Compile(context.Background(), conf.Schema, nil) require.NoError(t, err) require.NotNil(t, bundlemap) for _, m := range bundlemap.Queries { initialChecksum := m.Checksum - err := m.RefreshChecksum(context.Background(), schema, explorer.QueryMap(bundlemap.Queries).GetQuery) + err := m.RefreshChecksum(context.Background(), conf, explorer.QueryMap(bundlemap.Queries).GetQuery) require.NoError(t, err) assert.Equal(t, initialChecksum, m.Checksum, "checksum for %s changed", m.Mrn) } @@ -380,22 +379,19 @@ func TestBundleCompile_RemoveFailingQueries(t *testing.T) { - uid: check-1 mql: 1 == 2 - uid: check-2 - mql: muser.name != "" + mql: failme.name != "" queries: - uid: query-1 mql: 1 == 1 - uid: query-2 - mql: muser.name` + mql: failme.name` bundle := parseBundle(t, bundleStr) require.NotNil(t, bundle) - runtime := testutils.Local() - s := runtime.Schema() - delete(s.AllResources(), "muser") bundlemap, err := bundle.CompileExt(context.Background(), policy.BundleCompileConf{ - Schema: s, - Library: nil, - RemoveFailing: true, + CompilerConfig: conf, + Library: nil, + RemoveFailing: true, }) require.NoError(t, err) require.NotNil(t, bundlemap) diff --git a/policy/hub.go b/policy/hub.go index 6f6771f65..2484a7f12 100644 --- a/policy/hub.go +++ b/policy/hub.go @@ -96,7 +96,7 @@ func (s *LocalServices) PreparePolicy(ctx context.Context, policyObj *Policy, bu s.DataLake.GetValidatedPolicy, s.DataLake.GetQuery, bundle, - s.Runtime.Schema(), + s.NewCompilerConfig(), ) if err != nil { return nil, nil, err diff --git a/policy/mquery.go b/policy/mquery.go index 5ff5b0d1c..a72ed0fa6 100644 --- a/policy/mquery.go +++ b/policy/mquery.go @@ -4,12 +4,13 @@ package policy import ( + "sort" + "github.com/pkg/errors" "go.mondoo.com/cnquery/v9/checksums" "go.mondoo.com/cnquery/v9/explorer" - "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/mrn" - "sort" ) func RefreshMRN(ownerMRN string, existingMRN string, resource string, uid string) (string, error) { @@ -37,9 +38,9 @@ func RefreshMRN(ownerMRN string, existingMRN string, resource string, uid string return mrn.String(), nil } -func ChecksumAssetFilters(queries []*explorer.Mquery, schema llx.Schema) (string, error) { +func ChecksumAssetFilters(queries []*explorer.Mquery, conf mqlc.CompilerConfig) (string, error) { for i := range queries { - if _, err := queries[i].RefreshAsFilter("", schema); err != nil { + if _, err := queries[i].RefreshAsFilter("", conf); err != nil { return "", errors.New("failed to compile query: " + err.Error()) } } @@ -59,10 +60,10 @@ func ChecksumAssetFilters(queries []*explorer.Mquery, schema llx.Schema) (string // RefreshChecksums of all queries // Note: This method is used for testing purposes only. If you need it in other // places please make sure to implement the query lookup. -func (m *Mqueries) RefreshChecksums(schema llx.Schema, props map[string]explorer.PropertyRef) error { +func (m *Mqueries) RefreshChecksums(conf mqlc.CompilerConfig, props map[string]explorer.PropertyRef) error { queries := map[string]*explorer.Mquery{} for i := range m.Items { - if _, err := m.Items[i].RefreshChecksumAndType(queries, props, schema); err != nil { + if _, err := m.Items[i].RefreshChecksumAndType(queries, props, conf); err != nil { return err } } diff --git a/policy/mquery_test.go b/policy/mquery_test.go index a88235456..1b92f659f 100644 --- a/policy/mquery_test.go +++ b/policy/mquery_test.go @@ -4,15 +4,18 @@ package policy import ( - "go.mondoo.com/cnquery/v9/explorer" - "go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils" "testing" "github.com/stretchr/testify/assert" + "go.mondoo.com/cnquery/v9" + "go.mondoo.com/cnquery/v9/explorer" + "go.mondoo.com/cnquery/v9/mqlc" + "go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils" ) func TestMquery_Whitespaces(t *testing.T) { coreSchema := testutils.MustLoadSchema(testutils.SchemaProvider{Provider: "core"}) + conf := mqlc.NewConfig(coreSchema, cnquery.DefaultFeatures) mq := &explorer.Mquery{ Mql: " mondoo { version \n} \t\n ", @@ -22,11 +25,11 @@ func TestMquery_Whitespaces(t *testing.T) { Mql: "mondoo { version \n}", } - bundle, err := mq.RefreshChecksumAndType(nil, nil, coreSchema) + bundle, err := mq.RefreshChecksumAndType(nil, nil, conf) assert.NoError(t, err) assert.NotNil(t, bundle) - bundle, err = mqexpect.RefreshChecksumAndType(nil, nil, coreSchema) + bundle, err = mqexpect.RefreshChecksumAndType(nil, nil, conf) assert.NoError(t, err) assert.NotNil(t, bundle) @@ -35,6 +38,8 @@ func TestMquery_Whitespaces(t *testing.T) { func TestMquery_CodeIDs(t *testing.T) { coreSchema := testutils.MustLoadSchema(testutils.SchemaProvider{Provider: "core"}) + conf := mqlc.NewConfig(coreSchema, cnquery.DefaultFeatures) + mqAssetFilter := &explorer.Mquery{ Mql: "mondoo { version \n}", } @@ -43,10 +48,10 @@ func TestMquery_CodeIDs(t *testing.T) { Mql: "mondoo { version \n}", } - _, err := mqAssetFilter.RefreshAsFilter("//some.mrn", coreSchema) + _, err := mqAssetFilter.RefreshAsFilter("//some.mrn", conf) assert.NoError(t, err) - _, err = mqReg.RefreshChecksumAndType(nil, nil, coreSchema) + _, err = mqReg.RefreshChecksumAndType(nil, nil, conf) assert.NoError(t, err) assert.Equal(t, mqReg.CodeId, mqAssetFilter.CodeId) diff --git a/policy/policy.go b/policy/policy.go index 6ca4ff794..dfc9b6fbb 100644 --- a/policy/policy.go +++ b/policy/policy.go @@ -14,7 +14,7 @@ import ( "github.com/rs/zerolog/log" "go.mondoo.com/cnquery/v9/checksums" "go.mondoo.com/cnquery/v9/explorer" - "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/mrn" "go.mondoo.com/cnquery/v9/types" "google.golang.org/protobuf/proto" @@ -239,7 +239,7 @@ func (p *Policy) UpdateChecksums(ctx context.Context, getPolicy func(ctx context.Context, mrn string) (*Policy, error), getQuery func(ctx context.Context, mrn string) (*explorer.Mquery, error), bundle *PolicyBundleMap, - schema llx.Schema, + conf mqlc.CompilerConfig, ) error { // simplify the access if we don't have a bundle if bundle == nil { @@ -259,7 +259,7 @@ func (p *Policy) UpdateChecksums(ctx context.Context, // if we have local checksums set, we can take an optimized route; // if not, we have to update all checksums if p.LocalContentChecksum == "" || p.LocalExecutionChecksum == "" { - return p.updateAllChecksums(ctx, getPolicy, getQuery, bundle, schema) + return p.updateAllChecksums(ctx, getPolicy, getQuery, bundle, conf) } // otherwise we have local checksums and only need to recompute the @@ -292,7 +292,7 @@ func (p *Policy) UpdateChecksums(ctx context.Context, } if p.GraphContentChecksum == "" || p.GraphExecutionChecksum == "" { - err = p.UpdateChecksums(ctx, getPolicy, getQuery, bundle, schema) + err = p.UpdateChecksums(ctx, getPolicy, getQuery, bundle, conf) if err != nil { return err } @@ -313,7 +313,7 @@ func (p *Policy) updateAllChecksums(ctx context.Context, getPolicy func(ctx context.Context, mrn string) (*Policy, error), getQuery func(ctx context.Context, mrn string) (*explorer.Mquery, error), bundle *PolicyBundleMap, - schema llx.Schema, + conf mqlc.CompilerConfig, ) error { log.Trace().Str("policy", p.Mrn).Msg("update policy checksum") p.LocalContentChecksum = "" @@ -431,7 +431,7 @@ func (p *Policy) updateAllChecksums(ctx context.Context, if base, ok := bundle.Queries[check.Mrn]; ok { check = check.Merge(base) - if err := check.RefreshChecksum(ctx, schema, getQuery); err != nil { + if err := check.RefreshChecksum(ctx, conf, getQuery); err != nil { return err } } else if check.Checksum == "" { @@ -440,7 +440,7 @@ func (p *Policy) updateAllChecksums(ctx context.Context, } if x, err := getQuery(ctx, check.Mrn); err == nil { check = check.Merge(x) - if err := check.RefreshChecksum(ctx, schema, getQuery); err != nil { + if err := check.RefreshChecksum(ctx, conf, getQuery); err != nil { return err } } @@ -472,7 +472,7 @@ func (p *Policy) updateAllChecksums(ctx context.Context, if base, ok := bundle.Queries[query.Mrn]; ok { query = query.Merge(base) - if err := query.RefreshChecksum(ctx, schema, getQuery); err != nil { + if err := query.RefreshChecksum(ctx, conf, getQuery); err != nil { return err } } else if query.Checksum == "" { @@ -481,7 +481,7 @@ func (p *Policy) updateAllChecksums(ctx context.Context, } if x, err := getQuery(ctx, query.Mrn); err == nil { query = query.Merge(x) - if err := query.RefreshChecksum(ctx, schema, getQuery); err != nil { + if err := query.RefreshChecksum(ctx, conf, getQuery); err != nil { return err } } diff --git a/policy/policy_test.go b/policy/policy_test.go index fac52d089..bb2b60b58 100644 --- a/policy/policy_test.go +++ b/policy/policy_test.go @@ -10,18 +10,20 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.mondoo.com/cnquery/v9" "go.mondoo.com/cnquery/v9/explorer" - "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/mrn" "go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils" "go.mondoo.com/cnspec/v9/policy" ) -var schema llx.Schema +var conf mqlc.CompilerConfig func init() { runtime := testutils.Local() - schema = runtime.Schema() + schema := runtime.Schema() + conf = mqlc.NewConfig(schema, cnquery.DefaultFeatures) } func getChecksums(p *policy.Policy) map[string]string { @@ -95,12 +97,12 @@ func TestPolicyChecksums(t *testing.T) { ctx := context.Background() p := b.Policies[0] - _, err = b.Compile(ctx, schema, nil) + _, err = b.Compile(ctx, conf.Schema, nil) require.NoError(t, err) // regular checksum tests - err = p.UpdateChecksums(ctx, nil, nil, b.ToMap(), schema) + err = p.UpdateChecksums(ctx, nil, nil, b.ToMap(), conf) require.NoError(t, err, "computing initial checksums works") checksums := getChecksums(p) @@ -109,7 +111,7 @@ func TestPolicyChecksums(t *testing.T) { } p.InvalidateLocalChecksums() - err = p.UpdateChecksums(ctx, nil, nil, b.ToMap(), schema) + err = p.UpdateChecksums(ctx, nil, nil, b.ToMap(), conf) assert.NoError(t, err, "computing checksums again") assert.Equal(t, checksums, getChecksums(p), "recomputing yields same checksums") @@ -138,7 +140,7 @@ func TestPolicyChecksums(t *testing.T) { checksums = getChecksums(p) f(p) p.InvalidateLocalChecksums() - err = p.UpdateChecksums(ctx, nil, nil, b.ToMap(), schema) + err = p.UpdateChecksums(ctx, nil, nil, b.ToMap(), conf) assert.NoError(t, err, "computing checksums") testChecksums(t, []bool{false, true, false, true}, checksums, getChecksums(p)) }) @@ -158,8 +160,8 @@ func TestPolicyChecksums(t *testing.T) { Name: assetMrn.String(), } assetBundle := &policy.Bundle{Policies: []*policy.Policy{assetPolicy}} - assetBundle.Compile(ctx, schema, nil) - assetPolicy.UpdateChecksums(ctx, nil, nil, assetBundle.ToMap(), schema) + assetBundle.Compile(ctx, conf.Schema, nil) + assetPolicy.UpdateChecksums(ctx, nil, nil, assetBundle.ToMap(), conf) runContentTest(assetPolicy, "changing asset policy mrn", func(p *policy.Policy) { p.Mrn += "bling" @@ -192,7 +194,7 @@ func TestPolicyChecksums(t *testing.T) { checksums = getChecksums(p) f() p.InvalidateLocalChecksums() - err = p.UpdateChecksums(ctx, nil, nil, b.ToMap(), schema) + err = p.UpdateChecksums(ctx, nil, nil, b.ToMap(), conf) assert.NoError(t, err, "computing checksums") updated := getChecksums(p) testChecksums(t, []bool{false, false, false, false}, checksums, updated) @@ -229,9 +231,9 @@ queries: require.NoError(t, err) pInitial := bundleInitial.Policies[0] pInitial.InvalidateLocalChecksums() - initialBundleMap, err := bundleInitial.Compile(context.Background(), schema, nil) + initialBundleMap, err := bundleInitial.Compile(context.Background(), conf.Schema, nil) require.NoError(t, err) - err = pInitial.UpdateChecksums(context.Background(), nil, explorer.QueryMap(initialBundleMap.Queries).GetQuery, initialBundleMap, schema) + err = pInitial.UpdateChecksums(context.Background(), nil, explorer.QueryMap(initialBundleMap.Queries).GetQuery, initialBundleMap, conf) assert.NoError(t, err, "computing checksums") bundleUpdated, err := policy.BundleFromYAML([]byte(` @@ -260,9 +262,9 @@ queries: require.NoError(t, err) pUpdated := bundleUpdated.Policies[0] pUpdated.InvalidateLocalChecksums() - updatedBundleMap, err := bundleUpdated.Compile(context.Background(), schema, nil) + updatedBundleMap, err := bundleUpdated.Compile(context.Background(), conf.Schema, nil) require.NoError(t, err) - err = pUpdated.UpdateChecksums(context.Background(), nil, explorer.QueryMap(updatedBundleMap.Queries).GetQuery, updatedBundleMap, schema) + err = pUpdated.UpdateChecksums(context.Background(), nil, explorer.QueryMap(updatedBundleMap.Queries).GetQuery, updatedBundleMap, conf) assert.NoError(t, err, "computing checksums") require.NotEqual(t, pInitial.GraphExecutionChecksum, pUpdated.LocalContentChecksum) @@ -298,9 +300,9 @@ queries: require.NoError(t, err) pInitial := bundleInitial.Policies[0] pInitial.InvalidateLocalChecksums() - initialBundleMap, err := bundleInitial.Compile(context.Background(), schema, nil) + initialBundleMap, err := bundleInitial.Compile(context.Background(), conf.Schema, nil) require.NoError(t, err) - err = pInitial.UpdateChecksums(context.Background(), nil, explorer.QueryMap(initialBundleMap.Queries).GetQuery, initialBundleMap, schema) + err = pInitial.UpdateChecksums(context.Background(), nil, explorer.QueryMap(initialBundleMap.Queries).GetQuery, initialBundleMap, conf) assert.NoError(t, err, "computing checksums") bundleUpdated, err := policy.BundleFromYAML([]byte(` @@ -329,9 +331,9 @@ queries: require.NoError(t, err) pUpdated := bundleUpdated.Policies[0] pUpdated.InvalidateLocalChecksums() - updatedBundleMap, err := bundleUpdated.Compile(context.Background(), schema, nil) + updatedBundleMap, err := bundleUpdated.Compile(context.Background(), conf.Schema, nil) require.NoError(t, err) - err = pUpdated.UpdateChecksums(context.Background(), nil, explorer.QueryMap(updatedBundleMap.Queries).GetQuery, updatedBundleMap, schema) + err = pUpdated.UpdateChecksums(context.Background(), nil, explorer.QueryMap(updatedBundleMap.Queries).GetQuery, updatedBundleMap, conf) assert.NoError(t, err, "computing checksums") require.NotEqual(t, pInitial.GraphExecutionChecksum, pUpdated.LocalContentChecksum) @@ -369,7 +371,7 @@ queries: require.NoError(t, err) pInitial := bundleInitial.Policies[0] pInitial.InvalidateLocalChecksums() - _, err = bundleInitial.Compile(context.Background(), schema, nil) + _, err = bundleInitial.Compile(context.Background(), conf.Schema, nil) require.Contains(t, err.Error(), "variant cycle detected") } @@ -401,7 +403,7 @@ queries: require.NoError(t, err) pInitial := bundleInitial.Policies[0] pInitial.InvalidateLocalChecksums() - _, err = bundleInitial.Compile(context.Background(), schema, nil) + _, err = bundleInitial.Compile(context.Background(), conf.Schema, nil) require.Contains(t, err.Error(), "variant cycle detected") } } diff --git a/policy/resolver.go b/policy/resolver.go index cc1a6d381..b1434d883 100644 --- a/policy/resolver.go +++ b/policy/resolver.go @@ -18,6 +18,7 @@ import ( "go.mondoo.com/cnquery/v9/explorer" "go.mondoo.com/cnquery/v9/llx" "go.mondoo.com/cnquery/v9/logger" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/mrn" "go.mondoo.com/cnquery/v9/utils/sortx" "go.mondoo.com/ranger-rpc/codes" @@ -121,9 +122,10 @@ func (s *LocalServices) Unassign(ctx context.Context, assignment *PolicyAssignme func (s *LocalServices) SetProps(ctx context.Context, req *explorer.PropsReq) (*explorer.Empty, error) { // validate that the queries compile and fill in checksums + conf := s.NewCompilerConfig() for i := range req.Props { prop := req.Props[i] - code, err := prop.RefreshChecksumAndType(s.Runtime.Schema()) + code, err := prop.RefreshChecksumAndType(conf) if err != nil { return nil, err } @@ -359,6 +361,8 @@ type resolverCache struct { reportingJobsActive map[string]bool errors []*policyResolutionError bundleMap *PolicyBundleMap + + compilerConfig mqlc.CompilerConfig } type policyResolverCache struct { @@ -443,10 +447,11 @@ func (s *LocalServices) resolve(ctx context.Context, policyMrn string, assetFilt func (s *LocalServices) tryResolve(ctx context.Context, bundleMrn string, assetFilters []*explorer.Mquery) (*ResolvedPolicy, error) { logCtx := logger.FromContext(ctx) now := time.Now() + conf := s.NewCompilerConfig() // phase 1: resolve asset filters and see if we can find a cached policy // trying first with all asset filters - allFiltersChecksum, err := ChecksumAssetFilters(assetFilters, s.Runtime.Schema()) + allFiltersChecksum, err := ChecksumAssetFilters(assetFilters, conf) if err != nil { return nil, err } @@ -484,7 +489,7 @@ func (s *LocalServices) tryResolve(ctx context.Context, bundleMrn string, assetF assetFiltersMap[matchingFilters[i].CodeId] = struct{}{} } - assetFiltersChecksum, err := ChecksumAssetFilters(matchingFilters, s.Runtime.Schema()) + assetFiltersChecksum, err := ChecksumAssetFilters(matchingFilters, conf) if err != nil { return nil, err } @@ -519,6 +524,7 @@ func (s *LocalServices) tryResolve(ctx context.Context, bundleMrn string, assetF reportingJobsByMsum: map[string][]*ReportingJob{}, reportingJobsActive: map[string]bool{}, bundleMap: bundleMap, + compilerConfig: conf, } rjUUID := cache.relativeChecksum(policyObj.GraphExecutionChecksum) @@ -876,7 +882,7 @@ func (s *LocalServices) policyGroupToJobs(ctx context.Context, group *PolicyGrou if base, ok := cache.global.bundleMap.Queries[check.Mrn]; ok { check = check.Merge(base) err := check.RefreshChecksum(ctx, - s.Runtime.Schema(), + cache.global.compilerConfig, explorer.QueryMap(cache.global.bundleMap.Queries).GetQuery, ) if err != nil { @@ -935,7 +941,7 @@ func (s *LocalServices) policyGroupToJobs(ctx context.Context, group *PolicyGrou if base, ok := cache.global.bundleMap.Queries[query.Mrn]; ok { query = query.Merge(base) err := query.RefreshChecksum(ctx, - s.Runtime.Schema(), + cache.global.compilerConfig, explorer.QueryMap(cache.global.bundleMap.Queries).GetQuery, ) if err != nil { @@ -1190,7 +1196,7 @@ func (s *LocalServices) jobsToQueries(ctx context.Context, policyMrn string, cac } } - executionQuery, dataChecksum, err := mquery2executionQuery(prop, nil, map[string]string{}, collectorJob, false, s.Runtime.Schema()) + executionQuery, dataChecksum, err := mquery2executionQuery(prop, nil, map[string]string{}, collectorJob, false, cache.compilerConfig) if err != nil { return nil, nil, errors.New("resolver> failed to compile query for MRN " + prop.Mrn + ": " + err.Error()) } @@ -1205,7 +1211,7 @@ func (s *LocalServices) jobsToQueries(ctx context.Context, policyMrn string, cac } } - executionQuery, _, err := mquery2executionQuery(query, propTypes, propToChecksums, collectorJob, !isDataQuery, s.Runtime.Schema()) + executionQuery, _, err := mquery2executionQuery(query, propTypes, propToChecksums, collectorJob, !isDataQuery, cache.compilerConfig) if err != nil { return nil, nil, errors.New("resolver> failed to compile query for MRN " + query.Mrn + ": " + err.Error()) } @@ -1326,13 +1332,13 @@ func connectDatapointsToReportingJob(query *ExecutionQuery, job *ReportingJob, d } type queryLike interface { - Compile(props map[string]*llx.Primitive, schema llx.Schema) (*llx.CodeBundle, error) + Compile(props map[string]*llx.Primitive, conf mqlc.CompilerConfig) (*llx.CodeBundle, error) GetChecksum() string GetMql() string } -func mquery2executionQuery(query queryLike, props map[string]*llx.Primitive, propsToChecksums map[string]string, collectorJob *CollectorJob, isScoring bool, schema llx.Schema) (*ExecutionQuery, string, error) { - bundle, err := query.Compile(props, schema) +func mquery2executionQuery(query queryLike, props map[string]*llx.Primitive, propsToChecksums map[string]string, collectorJob *CollectorJob, isScoring bool, conf mqlc.CompilerConfig) (*ExecutionQuery, string, error) { + bundle, err := query.Compile(props, conf) if err != nil { return nil, "", err } diff --git a/policy/resolver_test.go b/policy/resolver_test.go index f3ba1139b..1060e58d6 100644 --- a/policy/resolver_test.go +++ b/policy/resolver_test.go @@ -293,7 +293,7 @@ policies: _, err = srv.SetBundle(ctx, b) require.NoError(t, err) - bundleMap, err := b.Compile(context.Background(), schema, nil) + bundleMap, err := b.Compile(context.Background(), conf.Schema, nil) require.NoError(t, err) rp, err := srv.Resolve(context.Background(), &policy.ResolveReq{ @@ -401,7 +401,7 @@ policies: // Recompute the checksums so that the resolved policy is invalidated assetPolicy.InvalidateAllChecksums() - assetPolicy.UpdateChecksums(context.Background(), srv.DataLake.GetRawPolicy, srv.DataLake.GetQuery, nil, schema) + assetPolicy.UpdateChecksums(context.Background(), srv.DataLake.GetRawPolicy, srv.DataLake.GetQuery, nil, conf) // Set the asset policy err = srv.DataLake.SetPolicy(context.Background(), assetPolicy, filters.Items) @@ -421,7 +421,7 @@ policies: // Recompute the checksums so that the resolved policy is invalidated assetPolicy.InvalidateAllChecksums() - assetPolicy.UpdateChecksums(context.Background(), srv.DataLake.GetRawPolicy, srv.DataLake.GetQuery, nil, schema) + assetPolicy.UpdateChecksums(context.Background(), srv.DataLake.GetRawPolicy, srv.DataLake.GetQuery, nil, conf) // Set the asset policy err = srv.DataLake.SetPolicy(context.Background(), assetPolicy, filters.Items) @@ -545,7 +545,7 @@ framework_maps: bundle, err := srv.GetBundle(context.Background(), &policy.Mrn{Mrn: "asset1"}) require.NoError(t, err) - bundleMap, err := bundle.Compile(context.Background(), schema, nil) + bundleMap, err := bundle.Compile(context.Background(), conf.Schema, nil) require.NoError(t, err) mrnToQueryId := map[string]string{} @@ -685,7 +685,7 @@ framework_maps: bundle, err := srv.GetBundle(context.Background(), &policy.Mrn{Mrn: "asset1"}) require.NoError(t, err) - bundleMap, err := bundle.Compile(context.Background(), schema, nil) + bundleMap, err := bundle.Compile(context.Background(), conf.Schema, nil) require.NoError(t, err) mrnToQueryId := map[string]string{} @@ -798,7 +798,7 @@ framework_maps: bundle, err := srv.GetBundle(context.Background(), &policy.Mrn{Mrn: "asset1"}) require.NoError(t, err) - bundleMap, err := bundle.Compile(context.Background(), schema, nil) + bundleMap, err := bundle.Compile(context.Background(), conf.Schema, nil) require.NoError(t, err) mrnToQueryId := map[string]string{} diff --git a/policy/scan/fetcher.go b/policy/scan/fetcher.go index 839d34d92..a9143d5d1 100644 --- a/policy/scan/fetcher.go +++ b/policy/scan/fetcher.go @@ -9,7 +9,7 @@ import ( "net/http" "github.com/pkg/errors" - "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnspec/v9" "go.mondoo.com/cnspec/v9/policy" ) @@ -24,7 +24,7 @@ func newFetcher() *fetcher { } } -func (f *fetcher) fetchBundles(ctx context.Context, schema llx.Schema, urls ...string) (*policy.Bundle, error) { +func (f *fetcher) fetchBundles(ctx context.Context, conf mqlc.CompilerConfig, urls ...string) (*policy.Bundle, error) { var res *policy.Bundle = &policy.Bundle{} for i := range urls { @@ -41,9 +41,9 @@ func (f *fetcher) fetchBundles(ctx context.Context, schema llx.Schema, urls ...s // need to generate MRNs for everything if _, err := cur.CompileExt(ctx, policy.BundleCompileConf{ - Schema: schema, - Library: nil, - RemoveFailing: true, + CompilerConfig: conf, + Library: nil, + RemoveFailing: true, }); err != nil { return nil, errors.Wrap(err, "failed to compile fetched bundle") } diff --git a/policy/scan/local_scanner.go b/policy/scan/local_scanner.go index a101d05f2..ad1b5d186 100644 --- a/policy/scan/local_scanner.go +++ b/policy/scan/local_scanner.go @@ -975,7 +975,8 @@ func (s *localAssetScanner) ensureBundle() error { return errors.New("cannot find any default policies for this asset (" + platform + ")") } - s.job.Bundle, err = s.fetcher.fetchBundles(s.job.Ctx, s.Runtime.Schema(), urls.Urls...) + conf := s.services.NewCompilerConfig() + s.job.Bundle, err = s.fetcher.fetchBundles(s.job.Ctx, conf, urls.Urls...) return err } diff --git a/policy/scan/local_scanner_test.go b/policy/scan/local_scanner_test.go index d591cbcf8..2bde10089 100644 --- a/policy/scan/local_scanner_test.go +++ b/policy/scan/local_scanner_test.go @@ -11,8 +11,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "go.mondoo.com/cnquery/v9" "go.mondoo.com/cnquery/v9/explorer" - "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/providers" "go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory" "go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils" @@ -117,15 +118,15 @@ func TestDefaultConfig(t *testing.T) { type LocalScannerSuite struct { suite.Suite - ctx context.Context - schema llx.Schema - job *Job + ctx context.Context + conf mqlc.CompilerConfig + job *Job } func (s *LocalScannerSuite) SetupSuite() { s.ctx = context.Background() runtime := testutils.Local() - s.schema = runtime.Schema() + s.conf = mqlc.NewConfig(runtime.Schema(), cnquery.DefaultFeatures) } func (s *LocalScannerSuite) BeforeTest(suiteName, testName string) { @@ -158,8 +159,8 @@ func (s *LocalScannerSuite) TestRunIncognito_SharedQuery() { s.Require().NoError(err) _, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{ - Schema: s.schema, - RemoveFailing: true, + CompilerConfig: s.conf, + RemoveFailing: true, }) s.Require().NoError(err) @@ -202,8 +203,8 @@ func (s *LocalScannerSuite) TestRunIncognito_ExceptionGroups() { s.Require().NoError(err) _, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{ - Schema: s.schema, - RemoveFailing: true, + CompilerConfig: s.conf, + RemoveFailing: true, }) s.Require().NoError(err) @@ -267,8 +268,8 @@ func (s *LocalScannerSuite) TestRunIncognito_ExceptionGroups_RejectedReview() { bundle.Policies[1].Groups[1].ReviewStatus = policy.ReviewStatus_REJECTED _, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{ - Schema: s.schema, - RemoveFailing: true, + CompilerConfig: s.conf, + RemoveFailing: true, }) s.Require().NoError(err) @@ -330,8 +331,8 @@ func (s *LocalScannerSuite) TestRunIncognito_QueryExceptions() { s.Require().NoError(err) _, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{ - Schema: s.schema, - RemoveFailing: true, + CompilerConfig: s.conf, + RemoveFailing: true, }) s.Require().NoError(err) @@ -392,8 +393,8 @@ func (s *LocalScannerSuite) TestRunIncognito_QueryExceptions_MultipleGroups() { s.Require().NoError(err) _, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{ - Schema: s.schema, - RemoveFailing: true, + CompilerConfig: s.conf, + RemoveFailing: true, }) s.Require().NoError(err) diff --git a/policy/services.go b/policy/services.go index af5e51e8a..6f3de4bb6 100644 --- a/policy/services.go +++ b/policy/services.go @@ -7,8 +7,10 @@ import ( "context" "net/http" + "go.mondoo.com/cnquery/v9" "go.mondoo.com/cnquery/v9/explorer" "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/ranger-rpc" ) @@ -82,3 +84,7 @@ func NewRemoteServices(addr string, auth []ranger.ClientPlugin, httpClient *http func (l *LocalServices) Schema() llx.Schema { return l.Runtime.Schema() } + +func (l *LocalServices) NewCompilerConfig() mqlc.CompilerConfig { + return mqlc.NewConfig(l.Schema(), cnquery.DefaultFeatures) +}