Skip to content

Commit

Permalink
🧹 add CompilerConfig on all compile-related calls
Browse files Browse the repository at this point in the history 
Align to mondoohq/cnquery#2510

Signed-off-by: Dominik Richter <[email protected]>
  • Loading branch information
@arlimus
arlimus committed Jan 7, 2024
1 parent ea1ed4d commit 4f52b36
Show file tree
Hide file tree
Showing 17 changed files with 134 additions and 109 deletions.
4 changes: 3 additions & 1 deletion apps/cnspec/cmd/scan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"go.mondoo.com/cnquery/v9/cli/inventoryloader"
"go.mondoo.com/cnquery/v9/cli/theme"
"go.mondoo.com/cnquery/v9/logger"
"go.mondoo.com/cnquery/v9/mqlc"
"go.mondoo.com/cnquery/v9/providers"
"go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin"
Expand Down Expand Up @@ -347,9 +348,10 @@ func (c *scanConfig) loadPolicies(ctx context.Context) error {
}

bundle.ConvertQuerypacks()
conf := mqlc.NewConfig(c.runtime.Schema(), cnquery.DefaultFeatures)

_, err = bundle.CompileExt(ctx, policy.BundleCompileConf{
Schema: c.runtime.Schema(),
CompilerConfig: conf,
// We don't care about failing queries for local runs. We may only
// process a subset of all the queries in the bundle. When we receive
// things from the server, upstream can filter things for us. But running
Expand Down
2 changes: 1 addition & 1 deletion internal/datalakes/inmemory/policyhub.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -538,7 +538,7 @@ func (db *Db) fixInvalidatedPolicy(ctx context.Context, wrap *wrapPolicy) error
func(ctx context.Context, mrn string) (*policy.Policy, error) { return db.GetValidatedPolicy(ctx, mrn) },
func(ctx context.Context, mrn string) (*explorer.Mquery, error) { return db.GetQuery(ctx, mrn) },
nil,
db.services.Schema(),
db.services.NewCompilerConfig(),
)

ok := db.cache.Set(dbIDPolicy+wrap.Policy.Mrn, *wrap, 2)
Expand Down
8 changes: 5 additions & 3 deletions internal/datalakes/inmemory/policyresolver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ func (db *Db) mutatePolicy(ctx context.Context, mrn string, actions map[string]e
func(ctx context.Context, mrn string) (*policy.Policy, error) { return db.GetValidatedPolicy(ctx, mrn) },
func(ctx context.Context, mrn string) (*explorer.Mquery, error) { return db.GetQuery(ctx, mrn) },
nil,
db.services.Schema(),
db.services.NewCompilerConfig(),
)
if err != nil {
return policyw, true, err
Expand Down Expand Up @@ -350,6 +350,8 @@ func (db *Db) refreshDependentAssetFilters(ctx context.Context, startPolicy wrap
}

for len(needsUpdate) > 0 {
conf := db.services.NewCompilerConfig()

for k, policyw := range needsUpdate {
err := db.refreshAssetFilters(ctx, &policyw)
if err != nil {
Expand All @@ -361,7 +363,7 @@ func (db *Db) refreshDependentAssetFilters(ctx context.Context, startPolicy wrap
func(ctx context.Context, mrn string) (*policy.Policy, error) { return db.GetValidatedPolicy(ctx, mrn) },
func(ctx context.Context, mrn string) (*explorer.Mquery, error) { return db.GetQuery(ctx, mrn) },
nil,
db.services.Schema(),
conf,
)
if err != nil {
return err
Expand Down Expand Up @@ -861,7 +863,7 @@ func (db *Db) SetProps(ctx context.Context, req *explorer.PropsReq) error {
func(ctx context.Context, mrn string) (*policy.Policy, error) { return db.GetValidatedPolicy(ctx, mrn) },
func(ctx context.Context, mrn string) (*explorer.Mquery, error) { return db.GetQuery(ctx, mrn) },
nil,
db.services.Schema(),
db.services.NewCompilerConfig(),
)
if err != nil {
return err
Expand Down
20 changes: 11 additions & 9 deletions policy/bundle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,12 @@ import (

"github.com/pkg/errors"
"github.com/rs/zerolog/log"
"go.mondoo.com/cnquery/v9"
"go.mondoo.com/cnquery/v9/checksums"
"go.mondoo.com/cnquery/v9/explorer"
"go.mondoo.com/cnquery/v9/llx"
"go.mondoo.com/cnquery/v9/logger"
"go.mondoo.com/cnquery/v9/mqlc"
"go.mondoo.com/cnquery/v9/mrn"
"go.mondoo.com/cnquery/v9/utils/multierr"
"sigs.k8s.io/yaml"
Expand Down Expand Up @@ -712,13 +714,13 @@ func topologicalSortQueriesDFS(queryMrn string, queriesMap map[string]*explorer.
// Compile a bundle. See CompileExt for a full description.
func (p *Bundle) Compile(ctx context.Context, schema llx.Schema, library Library) (*PolicyBundleMap, error) {
return p.CompileExt(ctx, BundleCompileConf{
Schema: schema,
Library: library,
CompilerConfig: mqlc.NewConfig(schema, cnquery.DefaultFeatures),
Library: library,
})
}

type BundleCompileConf struct {
Schema llx.Schema
mqlc.CompilerConfig

Check failure on line 723 in policy/bundle.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

undefined: mqlc.CompilerConfig

Check failure on line 723 in policy/bundle.go

View workflow job for this annotation

GitHub Actions / go-test 

undefined: mqlc.CompilerConfig
Library Library
RemoveFailing bool
}
Expand Down Expand Up @@ -811,7 +813,7 @@ func (p *Bundle) CompileExt(ctx context.Context, conf BundleCompileConf) (*Polic
if policy.ComputedFilters == nil || policy.ComputedFilters.Items == nil {
policy.ComputedFilters = &explorer.Filters{Items: map[string]*explorer.Mquery{}}
}
if err = policy.ComputedFilters.Compile(ownerMrn, cache.conf.Schema); err != nil {
if err = policy.ComputedFilters.Compile(ownerMrn, conf.CompilerConfig); err != nil {
return nil, multierr.Wrap(err, "failed to compile policy filters")
}

Expand All @@ -820,7 +822,7 @@ func (p *Bundle) CompileExt(ctx context.Context, conf BundleCompileConf) (*Polic
group := policy.Groups[i]

// When filters are initially added they haven't been compiled
if err = group.Filters.Compile(ownerMrn, cache.conf.Schema); err != nil {
if err = group.Filters.Compile(ownerMrn, conf.CompilerConfig); err != nil {
return nil, multierr.Wrap(err, "failed to compile policy group filters")
}

Expand Down Expand Up @@ -891,7 +893,7 @@ func (p *Bundle) CompileExt(ctx context.Context, conf BundleCompileConf) (*Polic
return nil, errors.New("failed to validate policy: " + err.Error())
}

err = bundleMap.ValidatePolicy(ctx, policy, cache.conf.Schema)
err = bundleMap.ValidatePolicy(ctx, policy, cache.conf.CompilerConfig)
if err != nil {
return nil, errors.New("failed to validate policy: " + err.Error())
}
Expand Down Expand Up @@ -1066,7 +1068,7 @@ func (c *bundleCache) precompileQuery(query *explorer.Mquery, policy *Policy) *e
}

// filters have no dependencies, so we can compile them early
if err := query.Filters.Compile(c.ownerMrn, c.conf.Schema); err != nil {
if err := query.Filters.Compile(c.ownerMrn, c.conf.CompilerConfig); err != nil {
c.errors = append(c.errors, errors.New("failed to compile filters for query "+query.Mrn))
return nil
}
Expand Down Expand Up @@ -1100,7 +1102,7 @@ func (c *bundleCache) precompileQuery(query *explorer.Mquery, policy *Policy) *e
// dependencies have been processed. Properties must be compiled. Connected
// queries may not be ready yet, but we have to have precompiled them.
func (c *bundleCache) compileQuery(query *explorer.Mquery) {
_, err := query.RefreshChecksumAndType(c.lookupQuery, c.lookupProp, c.conf.Schema)
_, err := query.RefreshChecksumAndType(c.lookupQuery, c.lookupProp, c.conf.CompilerConfig)
if err != nil {
if c.conf.RemoveFailing {
c.removeQueries[query.Mrn] = struct{}{}
Expand Down Expand Up @@ -1133,7 +1135,7 @@ func (c *bundleCache) compileProp(prop *explorer.Property) error {
name = m.Basename()
}

if _, err := prop.RefreshChecksumAndType(c.conf.Schema); err != nil {
if _, err := prop.RefreshChecksumAndType(c.conf.CompilerConfig); err != nil {
return err
}

Expand Down
9 changes: 5 additions & 4 deletions policy/bundle_map.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/pkg/errors"
"go.mondoo.com/cnquery/v9/explorer"
"go.mondoo.com/cnquery/v9/llx"
"go.mondoo.com/cnquery/v9/mqlc"
"go.mondoo.com/cnquery/v9/mrn"
"go.mondoo.com/cnquery/v9/utils/sortx"
)
Expand Down Expand Up @@ -173,13 +174,13 @@ func sortPolicies(p *Policy, bundle *PolicyBundleMap, indexer map[string]struct{
}

// ValidatePolicy against the given bundle
func (p *PolicyBundleMap) ValidatePolicy(ctx context.Context, policy *Policy, schema llx.Schema) error {
func (p *PolicyBundleMap) ValidatePolicy(ctx context.Context, policy *Policy, conf mqlc.CompilerConfig) error {

Check failure on line 177 in policy/bundle_map.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

undefined: mqlc.CompilerConfig

Check failure on line 177 in policy/bundle_map.go

View workflow job for this annotation

GitHub Actions / go-test 

undefined: mqlc.CompilerConfig
if !mrn.IsValid(policy.Mrn) {
return errors.New("policy MRN is not valid: " + policy.Mrn)
}

for i := range policy.Groups {
if err := p.validateGroup(ctx, policy.Groups[i], policy.Mrn, schema); err != nil {
if err := p.validateGroup(ctx, policy.Groups[i], policy.Mrn, conf); err != nil {
return err
}
}
Expand All @@ -195,7 +196,7 @@ func (p *PolicyBundleMap) ValidatePolicy(ctx context.Context, policy *Policy, sc
return nil
}

func (p *PolicyBundleMap) validateGroup(ctx context.Context, group *PolicyGroup, policyMrn string, schema llx.Schema) error {
func (p *PolicyBundleMap) validateGroup(ctx context.Context, group *PolicyGroup, policyMrn string, conf mqlc.CompilerConfig) error {

Check failure on line 199 in policy/bundle_map.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

undefined: mqlc.CompilerConfig

Check failure on line 199 in policy/bundle_map.go

View workflow job for this annotation

GitHub Actions / go-test 

undefined: mqlc.CompilerConfig
if group == nil {
return errors.New("spec cannot be nil")
}
Expand All @@ -204,7 +205,7 @@ func (p *PolicyBundleMap) validateGroup(ctx context.Context, group *PolicyGroup,
// since asset filters are run beforehand and don't make it into the report
// we don't store their code bundles separately
for _, query := range group.Filters.Items {
_, err := query.RefreshAsFilter(policyMrn, schema)
_, err := query.RefreshAsFilter(policyMrn, conf)
if err != nil {
return err
}
Expand Down
20 changes: 8 additions & 12 deletions policy/bundle_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/stretchr/testify/require"
"go.mondoo.com/cnquery/v9/explorer"
"go.mondoo.com/cnquery/v9/providers"
"go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils"
"go.mondoo.com/cnspec/v9/internal/datalakes/inmemory"
"go.mondoo.com/cnspec/v9/policy"
)
Expand Down Expand Up @@ -232,7 +231,7 @@ func TestBundleCompile(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, bundle)

bundlemap, err := bundle.Compile(context.Background(), schema, nil)
bundlemap, err := bundle.Compile(context.Background(), conf.Schema, nil)
require.NoError(t, err)
require.NotNil(t, bundlemap)

Expand Down Expand Up @@ -357,13 +356,13 @@ func TestStableMqueryChecksum(t *testing.T) {
require.NoError(t, err)
require.NotNil(t, bundle)

bundlemap, err := bundle.Compile(context.Background(), schema, nil)
bundlemap, err := bundle.Compile(context.Background(), conf.Schema, nil)
require.NoError(t, err)
require.NotNil(t, bundlemap)

for _, m := range bundlemap.Queries {
initialChecksum := m.Checksum
err := m.RefreshChecksum(context.Background(), schema, explorer.QueryMap(bundlemap.Queries).GetQuery)
err := m.RefreshChecksum(context.Background(), conf, explorer.QueryMap(bundlemap.Queries).GetQuery)
require.NoError(t, err)
assert.Equal(t, initialChecksum, m.Checksum, "checksum for %s changed", m.Mrn)
}
Expand All @@ -380,22 +379,19 @@ func TestBundleCompile_RemoveFailingQueries(t *testing.T) {
- uid: check-1
mql: 1 == 2
- uid: check-2
mql: muser.name != ""
mql: failme.name != ""
queries:
- uid: query-1
mql: 1 == 1
- uid: query-2
mql: muser.name`
mql: failme.name`

bundle := parseBundle(t, bundleStr)
require.NotNil(t, bundle)
runtime := testutils.Local()
s := runtime.Schema()
delete(s.AllResources(), "muser")
bundlemap, err := bundle.CompileExt(context.Background(), policy.BundleCompileConf{
Schema: s,
Library: nil,
RemoveFailing: true,
CompilerConfig: conf,
Library: nil,
RemoveFailing: true,
})
require.NoError(t, err)
require.NotNil(t, bundlemap)
Expand Down
2 changes: 1 addition & 1 deletion policy/hub.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func (s *LocalServices) PreparePolicy(ctx context.Context, policyObj *Policy, bu
s.DataLake.GetValidatedPolicy,
s.DataLake.GetQuery,
bundle,
s.Runtime.Schema(),
s.NewCompilerConfig(),
)
if err != nil {
return nil, nil, err
Expand Down
13 changes: 7 additions & 6 deletions policy/mquery.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@
package policy

import (
"sort"

"github.com/pkg/errors"
"go.mondoo.com/cnquery/v9/checksums"
"go.mondoo.com/cnquery/v9/explorer"
"go.mondoo.com/cnquery/v9/llx"
"go.mondoo.com/cnquery/v9/mqlc"
"go.mondoo.com/cnquery/v9/mrn"
"sort"
)

func RefreshMRN(ownerMRN string, existingMRN string, resource string, uid string) (string, error) {
Expand Down Expand Up @@ -37,9 +38,9 @@ func RefreshMRN(ownerMRN string, existingMRN string, resource string, uid string
return mrn.String(), nil
}

func ChecksumAssetFilters(queries []*explorer.Mquery, schema llx.Schema) (string, error) {
func ChecksumAssetFilters(queries []*explorer.Mquery, conf mqlc.CompilerConfig) (string, error) {

Check failure on line 41 in policy/mquery.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

undefined: mqlc.CompilerConfig

Check failure on line 41 in policy/mquery.go

View workflow job for this annotation

GitHub Actions / go-test 

undefined: mqlc.CompilerConfig
for i := range queries {
if _, err := queries[i].RefreshAsFilter("", schema); err != nil {
if _, err := queries[i].RefreshAsFilter("", conf); err != nil {
return "", errors.New("failed to compile query: " + err.Error())
}
}
Expand All @@ -59,10 +60,10 @@ func ChecksumAssetFilters(queries []*explorer.Mquery, schema llx.Schema) (string
// RefreshChecksums of all queries
// Note: This method is used for testing purposes only. If you need it in other
// places please make sure to implement the query lookup.
func (m *Mqueries) RefreshChecksums(schema llx.Schema, props map[string]explorer.PropertyRef) error {
func (m *Mqueries) RefreshChecksums(conf mqlc.CompilerConfig, props map[string]explorer.PropertyRef) error {

Check failure on line 63 in policy/mquery.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

undefined: mqlc.CompilerConfig

Check failure on line 63 in policy/mquery.go

View workflow job for this annotation

GitHub Actions / go-test 

undefined: mqlc.CompilerConfig
queries := map[string]*explorer.Mquery{}
for i := range m.Items {
if _, err := m.Items[i].RefreshChecksumAndType(queries, props, schema); err != nil {
if _, err := m.Items[i].RefreshChecksumAndType(queries, props, conf); err != nil {
return err
}
}
Expand Down
17 changes: 11 additions & 6 deletions policy/mquery_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,18 @@
package policy

import (
"go.mondoo.com/cnquery/v9/explorer"
"go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils"
"testing"

"github.com/stretchr/testify/assert"
"go.mondoo.com/cnquery/v9"
"go.mondoo.com/cnquery/v9/explorer"
"go.mondoo.com/cnquery/v9/mqlc"
"go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils"
)

func TestMquery_Whitespaces(t *testing.T) {
coreSchema := testutils.MustLoadSchema(testutils.SchemaProvider{Provider: "core"})
conf := mqlc.NewConfig(coreSchema, cnquery.DefaultFeatures)

mq := &explorer.Mquery{
Mql: " mondoo { version \n} \t\n ",
Expand All @@ -22,11 +25,11 @@ func TestMquery_Whitespaces(t *testing.T) {
Mql: "mondoo { version \n}",
}

bundle, err := mq.RefreshChecksumAndType(nil, nil, coreSchema)
bundle, err := mq.RefreshChecksumAndType(nil, nil, conf)
assert.NoError(t, err)
assert.NotNil(t, bundle)

bundle, err = mqexpect.RefreshChecksumAndType(nil, nil, coreSchema)
bundle, err = mqexpect.RefreshChecksumAndType(nil, nil, conf)
assert.NoError(t, err)
assert.NotNil(t, bundle)

Expand All @@ -35,6 +38,8 @@ func TestMquery_Whitespaces(t *testing.T) {

func TestMquery_CodeIDs(t *testing.T) {
coreSchema := testutils.MustLoadSchema(testutils.SchemaProvider{Provider: "core"})
conf := mqlc.NewConfig(coreSchema, cnquery.DefaultFeatures)

mqAssetFilter := &explorer.Mquery{
Mql: "mondoo { version \n}",
}
Expand All @@ -43,10 +48,10 @@ func TestMquery_CodeIDs(t *testing.T) {
Mql: "mondoo { version \n}",
}

_, err := mqAssetFilter.RefreshAsFilter("//some.mrn", coreSchema)
_, err := mqAssetFilter.RefreshAsFilter("//some.mrn", conf)
assert.NoError(t, err)

_, err = mqReg.RefreshChecksumAndType(nil, nil, coreSchema)
_, err = mqReg.RefreshChecksumAndType(nil, nil, conf)
assert.NoError(t, err)

assert.Equal(t, mqReg.CodeId, mqAssetFilter.CodeId)
Expand Down
Loading

0 comments on commit 4f52b36

Please sign in to comment.