🧹 add CompilerConfig on all compile-related calls (#2510)

Until now, the only thing developers can configure when it comes to
compile-related calls is the schema.

However, there are use-cases where we may want to configure other things
in the compiler, like:
- which feature-flags are enabled/disabled
- use of stats

I ran into the last use-case myself, which would have caused large
copy-paste rewrites of the Bundle-compiler, just in order to get access
to all the stats.

At this point it is a more comprehensive experience to not move the
schema around, but instead wire the compiler config directly through all
these calls. This means that more future-facing features will be
configurable and that a chain of calls (like `Bundle.Compile`) uses a
consistent set of compiler configs.

Signed-off-by: Dominik Richter <[email protected]>

apps/cnquery/cmd/scan.go

@@ -21,6 +21,7 @@ import (
 	"go.mondoo.com/cnquery/v9/cli/theme"
 	"go.mondoo.com/cnquery/v9/explorer"
 	"go.mondoo.com/cnquery/v9/explorer/scan"
+	"go.mondoo.com/cnquery/v9/mqlc"
 	"go.mondoo.com/cnquery/v9/providers"
 	"go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory"
 	"go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin"
@@ -282,8 +283,9 @@ func (c *scanConfig) loadBundles() error {
 			return err
 		}
 
+		conf := mqlc.NewConfig(c.runtime.Schema(), cnquery.DefaultFeatures)
 		_, err = bundle.CompileExt(context.Background(), explorer.BundleCompileConf{
-			Schema: c.runtime.Schema(),
+			CompilerConfig: conf,
 			// We don't care about failing queries for local runs. We may only
 			// process a subset of all the queries in the bundle. When we receive
 			// things from the server, upstream can filter things for us. But running

explorer/bundle.go

@@ -13,8 +13,10 @@ import (
 
 	"github.com/rs/zerolog/log"
 	"github.com/segmentio/ksuid"
+	"go.mondoo.com/cnquery/v9"
 	"go.mondoo.com/cnquery/v9/checksums"
 	llx "go.mondoo.com/cnquery/v9/llx"
+	"go.mondoo.com/cnquery/v9/mqlc"
 	"go.mondoo.com/cnquery/v9/mrn"
 	"go.mondoo.com/cnquery/v9/utils/multierr"
 	"sigs.k8s.io/yaml"
@@ -212,12 +214,12 @@ func (p *Bundle) AddBundle(other *Bundle) error {
 // Compile a bundle. See CompileExt for a full description.
 func (p *Bundle) Compile(ctx context.Context, schema llx.Schema) (*BundleMap, error) {
 	return p.CompileExt(ctx, BundleCompileConf{
-		Schema: schema,
+		CompilerConfig: mqlc.NewConfig(schema, cnquery.DefaultFeatures),
 	})
 }
 
 type BundleCompileConf struct {
-	Schema        llx.Schema
+	mqlc.CompilerConfig
 	RemoveFailing bool
 }
 
@@ -265,7 +267,7 @@ func (bundle *Bundle) CompileExt(ctx context.Context, conf BundleCompileConf) (*
 			return nil, multierr.Wrap(err, "failed to refresh query pack "+pack.Mrn)
 		}
 
-		if err = pack.Filters.Compile(ownerMrn, conf.Schema); err != nil {
+		if err = pack.Filters.Compile(ownerMrn, conf.CompilerConfig); err != nil {
 			return nil, multierr.Wrap(err, "failed to compile querypack filters")
 		}
 		pack.ComputedFilters.AddFilters(pack.Filters)
@@ -278,7 +280,7 @@ func (bundle *Bundle) CompileExt(ctx context.Context, conf BundleCompileConf) (*
 			group := pack.Groups[i]
 
 			// When filters are initially added they haven't been compiled
-			if err = group.Filters.Compile(ownerMrn, conf.Schema); err != nil {
+			if err = group.Filters.Compile(ownerMrn, conf.CompilerConfig); err != nil {
 				return nil, multierr.Wrap(err, "failed to compile querypack filters")
 			}
 			pack.ComputedFilters.AddFilters(group.Filters)
@@ -419,7 +421,7 @@ func (c *bundleCache) precompileQuery(query *Mquery, pack *QueryPack) {
 	}
 
 	// filters have no dependencies, so we can compile them early
-	if err := query.Filters.Compile(c.ownerMrn, c.conf.Schema); err != nil {
+	if err := query.Filters.Compile(c.ownerMrn, c.conf.CompilerConfig); err != nil {
 		c.errors = append(c.errors, errors.New("failed to compile filters for query "+query.Mrn))
 		return
 	}
@@ -450,7 +452,7 @@ func (c *bundleCache) precompileQuery(query *Mquery, pack *QueryPack) {
 // dependencies have been processed. Properties must be compiled. Connected
 // queries may not be ready yet, but we have to have precompiled them.
 func (c *bundleCache) compileQuery(query *Mquery) {
-	_, err := query.RefreshChecksumAndType(c.lookupQuery, c.lookupProp, c.conf.Schema)
+	_, err := query.RefreshChecksumAndType(c.lookupQuery, c.lookupProp, c.conf.CompilerConfig)
 	if err != nil {
 		if c.conf.RemoveFailing {
 			c.removeQueries[query.Mrn] = struct{}{}
@@ -483,7 +485,7 @@ func (c *bundleCache) compileProp(prop *Property) error {
 		name = m.Basename()
 	}
 
-	if _, err := prop.RefreshChecksumAndType(c.conf.Schema); err != nil {
+	if _, err := prop.RefreshChecksumAndType(c.conf.CompilerConfig); err != nil {
 		return err
 	}
 

explorer/bundle_test.go

@@ -10,10 +10,15 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.mondoo.com/cnquery/v9"
+	"go.mondoo.com/cnquery/v9/mqlc"
 	"go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils"
 )
 
-var mock = testutils.LinuxMock()
+var (
+	mock = testutils.LinuxMock()
+	conf = mqlc.NewConfig(mock.Schema(), cnquery.DefaultFeatures)
+)
 
 func TestBundleLoad(t *testing.T) {
 	t.Run("load bundle from file", func(t *testing.T) {
@@ -56,8 +61,8 @@ func TestFilterQueriesWontCompile(t *testing.T) {
 	b2, err := BundleFromYAML([]byte(failingVariant))
 	require.NoError(t, err)
 	_, err2 := b2.CompileExt(context.Background(), BundleCompileConf{
-		Schema:        mock.Schema(),
-		RemoveFailing: false,
+		CompilerConfig: conf,
+		RemoveFailing:  false,
 	})
 	require.Error(t, err2)
 }
@@ -66,8 +71,8 @@ func TestFilterQueriesIgnoreError(t *testing.T) {
 	b, err := BundleFromYAML([]byte(failingVariant))
 	require.NoError(t, err)
 	bmap, err := b.CompileExt(context.Background(), BundleCompileConf{
-		Schema:        mock.Schema(),
-		RemoveFailing: true,
+		CompilerConfig: conf,
+		RemoveFailing:  true,
 	})
 	require.NoError(t, err)
 	require.NotNil(t, bmap)

explorer/executor/executor.go

@@ -14,6 +14,7 @@ import (
 	"go.mondoo.com/cnquery/v9/cli/progress"
 	"go.mondoo.com/cnquery/v9/explorer"
 	"go.mondoo.com/cnquery/v9/llx"
+	"go.mondoo.com/cnquery/v9/mqlc"
 	"go.mondoo.com/cnquery/v9/utils/multierr"
 )
 
@@ -36,9 +37,10 @@ func RunExecutionJob(
 func ExecuteFilterQueries(runtime llx.Runtime, queries []*explorer.Mquery, timeout time.Duration) ([]*explorer.Mquery, []error) {
 	equeries := map[string]*explorer.ExecutionQuery{}
 	mqueries := map[string]*explorer.Mquery{}
+	conf := mqlc.NewConfig(runtime.Schema(), cnquery.DefaultFeatures)
 	for i := range queries {
 		query := queries[i]
-		code, err := query.Compile(nil, runtime.Schema())
+		code, err := query.Compile(nil, conf)
 		// Errors for filter queries are common when they reference resources for
 		// providers that are not found on the system.
 		if err != nil {

explorer/filters.go

@@ -12,7 +12,7 @@ import (
 	"strings"
 
 	"go.mondoo.com/cnquery/v9/checksums"
-	llx "go.mondoo.com/cnquery/v9/llx"
+	"go.mondoo.com/cnquery/v9/mqlc"
 	"go.mondoo.com/cnquery/v9/utils/multierr"
 )
 
@@ -114,14 +114,17 @@ func (s *Filters) UnmarshalJSON(data []byte) error {
 	return json.Unmarshal(data, (*tmp)(s))
 }
 
-func (s *Filters) Compile(ownerMRN string, schema llx.Schema) error {
+func (s *Filters) Compile(ownerMRN string, conf mqlc.CompilerConfig) error {
 	if s == nil || len(s.Items) == 0 {
 		return nil
 	}
 
 	res := make(map[string]*Mquery, len(s.Items))
 	for _, query := range s.Items {
-		query.RefreshAsFilter(ownerMRN, schema)
+		_, err := query.RefreshAsFilter(ownerMRN, conf)
+		if err != nil {
+			return err
+		}
 
 		if _, ok := res[query.CodeId]; ok {
 			continue

explorer/impact.go

@@ -11,6 +11,24 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+func (v *Impact) HumanReadable() string {
+	if v.Value == nil {
+		return "unknown"
+	}
+	switch {
+	case v.Value.Value >= 90:
+		return "critical"
+	case v.Value.Value >= 70:
+		return "high"
+	case v.Value.Value >= 40:
+		return "medium"
+	case v.Value.Value > 0:
+		return "low"
+	default:
+		return "info"
+	}
+}
+
 func (v *Impact) AddBase(base *Impact) {
 	if base == nil {
 		return

explorer/mquery.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 
 	"github.com/rs/zerolog/log"
-	"go.mondoo.com/cnquery/v9"
 	"go.mondoo.com/cnquery/v9/checksums"
 	llx "go.mondoo.com/cnquery/v9/llx"
 	"go.mondoo.com/cnquery/v9/mqlc"
@@ -24,7 +23,7 @@ import (
 
 // Compile a given query and return the bundle. Both v1 and v2 versions are compiled.
 // Both versions will be given the same code id.
-func (m *Mquery) Compile(props map[string]*llx.Primitive, schema llx.Schema) (*llx.CodeBundle, error) {
+func (m *Mquery) Compile(props map[string]*llx.Primitive, conf mqlc.CompilerConfig) (*llx.CodeBundle, error) {
 	if m.Mql == "" {
 		if m.Query == "" {
 			return nil, errors.New("query is not implemented '" + m.Mrn + "'")
@@ -33,7 +32,7 @@ func (m *Mquery) Compile(props map[string]*llx.Primitive, schema llx.Schema) (*l
 		m.Query = ""
 	}
 
-	v2Code, err := mqlc.Compile(m.Mql, props, mqlc.NewConfig(schema, cnquery.DefaultFeatures))
+	v2Code, err := mqlc.Compile(m.Mql, props, conf)
 	if err != nil {
 		return nil, err
 	}
@@ -111,7 +110,7 @@ func (m *ObjectRef) RefreshMRN(ownerMRN string) error {
 // since their internal checksum is not stored in this query.
 func (m *Mquery) RefreshChecksum(
 	ctx context.Context,
-	schema llx.Schema,
+	conf mqlc.CompilerConfig,
 	getQuery func(ctx context.Context, mrn string) (*Mquery, error),
 ) error {
 	c := checksums.New.
@@ -125,7 +124,7 @@ func (m *Mquery) RefreshChecksum(
 
 	for i := range m.Props {
 		prop := m.Props[i]
-		if _, err := prop.RefreshChecksumAndType(schema); err != nil {
+		if _, err := prop.RefreshChecksumAndType(conf); err != nil {
 			return err
 		}
 		if prop.Checksum == "" {
@@ -137,7 +136,7 @@ func (m *Mquery) RefreshChecksum(
 	for i := range m.Variants {
 		ref := m.Variants[i]
 		if q, err := getQuery(context.Background(), ref.Mrn); err == nil {
-			if err := q.RefreshChecksum(ctx, schema, getQuery); err != nil {
+			if err := q.RefreshChecksum(ctx, conf, getQuery); err != nil {
 				return err
 			}
 			if q.Checksum == "" {
@@ -160,7 +159,10 @@ func (m *Mquery) RefreshChecksum(
 					Str("mql", m.Mql).
 					Str("filter", query.Mql).
 					Msg("refresh checksum on filter of query , which should have been pre-compiled")
-				query.RefreshAsFilter(m.Mrn, schema)
+				_, err := query.RefreshAsFilter(m.Mrn, conf)
+				if err != nil {
+					return multierr.Wrap(err, "cannot refresh checksum for query, failed to compile")
+				}
 				if query.Checksum == "" {
 					return errors.New("cannot refresh checksum for query, its filters were not compiled")
 				}
@@ -200,8 +202,8 @@ func (m *Mquery) RefreshChecksum(
 }
 
 // RefreshChecksumAndType by compiling the query and updating the Checksum field
-func (m *Mquery) RefreshChecksumAndType(queries map[string]*Mquery, props map[string]PropertyRef, schema llx.Schema) (*llx.CodeBundle, error) {
-	return m.refreshChecksumAndType(queries, props, schema)
+func (m *Mquery) RefreshChecksumAndType(queries map[string]*Mquery, props map[string]PropertyRef, conf mqlc.CompilerConfig) (*llx.CodeBundle, error) {
+	return m.refreshChecksumAndType(queries, props, conf)
 }
 
 type QueryMap map[string]*Mquery
@@ -218,7 +220,7 @@ func (m QueryMap) GetQuery(ctx context.Context, mrn string) (*Mquery, error) {
 	return res, nil
 }
 
-func (m *Mquery) refreshChecksumAndType(queries map[string]*Mquery, props map[string]PropertyRef, schema llx.Schema) (*llx.CodeBundle, error) {
+func (m *Mquery) refreshChecksumAndType(queries map[string]*Mquery, props map[string]PropertyRef, conf mqlc.CompilerConfig) (*llx.CodeBundle, error) {
 	localProps := map[string]*llx.Primitive{}
 	for i := range m.Props {
 		prop := m.Props[i]
@@ -246,10 +248,10 @@ func (m *Mquery) refreshChecksumAndType(queries map[string]*Mquery, props map[st
 		if m.Mql != "" {
 			log.Warn().Str("msn", m.Mrn).Msg("a composed query is trying to define an mql snippet, which will be ignored")
 		}
-		return nil, m.RefreshChecksum(context.Background(), schema, QueryMap(queries).GetQuery)
+		return nil, m.RefreshChecksum(context.Background(), conf, QueryMap(queries).GetQuery)
 	}
 
-	bundle, err := m.Compile(localProps, schema)
+	bundle, err := m.Compile(localProps, conf)
 	if err != nil {
 		return bundle, multierr.Wrap(err, "failed to compile query '"+m.Mql+"'")
 	}
@@ -275,12 +277,12 @@ func (m *Mquery) refreshChecksumAndType(queries map[string]*Mquery, props map[st
 		m.Type = string(types.Any)
 	}
 
-	return bundle, m.RefreshChecksum(context.Background(), schema, QueryMap(queries).GetQuery)
+	return bundle, m.RefreshChecksum(context.Background(), conf, QueryMap(queries).GetQuery)
 }
 
 // RefreshAsFilter filters treats this query as an asset filter and sets its Mrn, Title, and Checksum
-func (m *Mquery) RefreshAsFilter(mrn string, schema llx.Schema) (*llx.CodeBundle, error) {
-	bundle, err := m.refreshChecksumAndType(nil, nil, schema)
+func (m *Mquery) RefreshAsFilter(mrn string, conf mqlc.CompilerConfig) (*llx.CodeBundle, error) {
+	bundle, err := m.refreshChecksumAndType(nil, nil, conf)
 	if err != nil {
 		return bundle, err
 	}
@@ -295,7 +297,7 @@ func (m *Mquery) RefreshAsFilter(mrn string, schema llx.Schema) (*llx.CodeBundle
 	}
 
 	if checksumInvalidated {
-		if err := m.RefreshChecksum(context.Background(), schema, nil); err != nil {
+		if err := m.RefreshChecksum(context.Background(), conf, nil); err != nil {
 			return nil, err
 		}
 	}
@@ -468,9 +470,9 @@ func (r *Remediation) MarshalJSON() ([]byte, error) {
 	return json.Marshal(r.Items)
 }
 
-func ChecksumFilters(queries []*Mquery, schema llx.Schema) (string, error) {
+func ChecksumFilters(queries []*Mquery, conf mqlc.CompilerConfig) (string, error) {
 	for i := range queries {
-		if _, err := queries[i].refreshChecksumAndType(nil, nil, schema); err != nil {
+		if _, err := queries[i].refreshChecksumAndType(nil, nil, conf); err != nil {
 			return "", multierr.Wrap(err, "failed to compile query")
 		}
 	}

explorer/mquery_test.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.mondoo.com/cnquery/v9"
+	"go.mondoo.com/cnquery/v9/mqlc"
 	"go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils"
 )
 
@@ -20,13 +22,14 @@ func TestMquery_RefreshAsAssetFilterStableChecksum(t *testing.T) {
 	}
 
 	x := testutils.LinuxMock()
+	conf := mqlc.NewConfig(x.Schema(), cnquery.DefaultFeatures)
 
-	_, err := m.RefreshAsFilter("//owner/me", x.Schema())
+	_, err := m.RefreshAsFilter("//owner/me", conf)
 	require.NoError(t, err)
 	assert.Equal(t, "//owner/me/filter/"+m.CodeId, m.Mrn)
 
 	cs := m.Checksum
-	_, err = m.RefreshAsFilter("//owner/me", x.Schema())
+	_, err = m.RefreshAsFilter("//owner/me", conf)
 	require.NoError(t, err)
 	assert.Equal(t, cs, m.Checksum)
 }
@@ -46,9 +49,10 @@ func TestMquery_Refresh(t *testing.T) {
 	assert.Empty(t, a.Props[0].Uid)
 
 	x := testutils.LinuxMock()
+	conf := mqlc.NewConfig(x.Schema(), cnquery.DefaultFeatures)
 	err = a.RefreshChecksum(
 		context.Background(),
-		x.Schema(),
+		conf,
 		func(ctx context.Context, mrn string) (*Mquery, error) {
 			return nil, nil
 		},