diff --git a/apps/cnquery/cmd/scan.go b/apps/cnquery/cmd/scan.go index b279b3c486..81d388231b 100644 --- a/apps/cnquery/cmd/scan.go +++ b/apps/cnquery/cmd/scan.go @@ -21,6 +21,7 @@ import ( "go.mondoo.com/cnquery/v9/cli/theme" "go.mondoo.com/cnquery/v9/explorer" "go.mondoo.com/cnquery/v9/explorer/scan" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/providers" "go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory" "go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin" @@ -282,8 +283,9 @@ func (c *scanConfig) loadBundles() error { return err } + conf := mqlc.NewConfig(c.runtime.Schema(), cnquery.DefaultFeatures) _, err = bundle.CompileExt(context.Background(), explorer.BundleCompileConf{ - Schema: c.runtime.Schema(), + CompilerConfig: conf, // We don't care about failing queries for local runs. We may only // process a subset of all the queries in the bundle. When we receive // things from the server, upstream can filter things for us. But running diff --git a/explorer/bundle.go b/explorer/bundle.go index 15727010ce..92886a0ee2 100644 --- a/explorer/bundle.go +++ b/explorer/bundle.go @@ -13,8 +13,10 @@ import ( "github.com/rs/zerolog/log" "github.com/segmentio/ksuid" + "go.mondoo.com/cnquery/v9" "go.mondoo.com/cnquery/v9/checksums" llx "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/mrn" "go.mondoo.com/cnquery/v9/utils/multierr" "sigs.k8s.io/yaml" @@ -212,12 +214,12 @@ func (p *Bundle) AddBundle(other *Bundle) error { // Compile a bundle. See CompileExt for a full description. func (p *Bundle) Compile(ctx context.Context, schema llx.Schema) (*BundleMap, error) { return p.CompileExt(ctx, BundleCompileConf{ - Schema: schema, + CompilerConfig: mqlc.NewConfig(schema, cnquery.DefaultFeatures), }) } type BundleCompileConf struct { - Schema llx.Schema + mqlc.CompilerConfig RemoveFailing bool } @@ -265,7 +267,7 @@ func (bundle *Bundle) CompileExt(ctx context.Context, conf BundleCompileConf) (* return nil, multierr.Wrap(err, "failed to refresh query pack "+pack.Mrn) } - if err = pack.Filters.Compile(ownerMrn, conf.Schema); err != nil { + if err = pack.Filters.Compile(ownerMrn, conf.CompilerConfig); err != nil { return nil, multierr.Wrap(err, "failed to compile querypack filters") } pack.ComputedFilters.AddFilters(pack.Filters) @@ -278,7 +280,7 @@ func (bundle *Bundle) CompileExt(ctx context.Context, conf BundleCompileConf) (* group := pack.Groups[i] // When filters are initially added they haven't been compiled - if err = group.Filters.Compile(ownerMrn, conf.Schema); err != nil { + if err = group.Filters.Compile(ownerMrn, conf.CompilerConfig); err != nil { return nil, multierr.Wrap(err, "failed to compile querypack filters") } pack.ComputedFilters.AddFilters(group.Filters) @@ -419,7 +421,7 @@ func (c *bundleCache) precompileQuery(query *Mquery, pack *QueryPack) { } // filters have no dependencies, so we can compile them early - if err := query.Filters.Compile(c.ownerMrn, c.conf.Schema); err != nil { + if err := query.Filters.Compile(c.ownerMrn, c.conf.CompilerConfig); err != nil { c.errors = append(c.errors, errors.New("failed to compile filters for query "+query.Mrn)) return } @@ -450,7 +452,7 @@ func (c *bundleCache) precompileQuery(query *Mquery, pack *QueryPack) { // dependencies have been processed. Properties must be compiled. Connected // queries may not be ready yet, but we have to have precompiled them. func (c *bundleCache) compileQuery(query *Mquery) { - _, err := query.RefreshChecksumAndType(c.lookupQuery, c.lookupProp, c.conf.Schema) + _, err := query.RefreshChecksumAndType(c.lookupQuery, c.lookupProp, c.conf.CompilerConfig) if err != nil { if c.conf.RemoveFailing { c.removeQueries[query.Mrn] = struct{}{} @@ -483,7 +485,7 @@ func (c *bundleCache) compileProp(prop *Property) error { name = m.Basename() } - if _, err := prop.RefreshChecksumAndType(c.conf.Schema); err != nil { + if _, err := prop.RefreshChecksumAndType(c.conf.CompilerConfig); err != nil { return err } diff --git a/explorer/bundle_test.go b/explorer/bundle_test.go index fb7a7b15cc..f04017d98c 100644 --- a/explorer/bundle_test.go +++ b/explorer/bundle_test.go @@ -10,10 +10,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.mondoo.com/cnquery/v9" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils" ) -var mock = testutils.LinuxMock() +var ( + mock = testutils.LinuxMock() + conf = mqlc.NewConfig(mock.Schema(), cnquery.DefaultFeatures) +) func TestBundleLoad(t *testing.T) { t.Run("load bundle from file", func(t *testing.T) { @@ -56,8 +61,8 @@ func TestFilterQueriesWontCompile(t *testing.T) { b2, err := BundleFromYAML([]byte(failingVariant)) require.NoError(t, err) _, err2 := b2.CompileExt(context.Background(), BundleCompileConf{ - Schema: mock.Schema(), - RemoveFailing: false, + CompilerConfig: conf, + RemoveFailing: false, }) require.Error(t, err2) } @@ -66,8 +71,8 @@ func TestFilterQueriesIgnoreError(t *testing.T) { b, err := BundleFromYAML([]byte(failingVariant)) require.NoError(t, err) bmap, err := b.CompileExt(context.Background(), BundleCompileConf{ - Schema: mock.Schema(), - RemoveFailing: true, + CompilerConfig: conf, + RemoveFailing: true, }) require.NoError(t, err) require.NotNil(t, bmap) diff --git a/explorer/executor/executor.go b/explorer/executor/executor.go index c1e2844335..ffe979c98c 100644 --- a/explorer/executor/executor.go +++ b/explorer/executor/executor.go @@ -14,6 +14,7 @@ import ( "go.mondoo.com/cnquery/v9/cli/progress" "go.mondoo.com/cnquery/v9/explorer" "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/utils/multierr" ) @@ -36,9 +37,10 @@ func RunExecutionJob( func ExecuteFilterQueries(runtime llx.Runtime, queries []*explorer.Mquery, timeout time.Duration) ([]*explorer.Mquery, []error) { equeries := map[string]*explorer.ExecutionQuery{} mqueries := map[string]*explorer.Mquery{} + conf := mqlc.NewConfig(runtime.Schema(), cnquery.DefaultFeatures) for i := range queries { query := queries[i] - code, err := query.Compile(nil, runtime.Schema()) + code, err := query.Compile(nil, conf) // Errors for filter queries are common when they reference resources for // providers that are not found on the system. if err != nil { diff --git a/explorer/filters.go b/explorer/filters.go index 551e39f6f7..9dee5f9a2b 100644 --- a/explorer/filters.go +++ b/explorer/filters.go @@ -12,7 +12,7 @@ import ( "strings" "go.mondoo.com/cnquery/v9/checksums" - llx "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/utils/multierr" ) @@ -114,14 +114,17 @@ func (s *Filters) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, (*tmp)(s)) } -func (s *Filters) Compile(ownerMRN string, schema llx.Schema) error { +func (s *Filters) Compile(ownerMRN string, conf mqlc.CompilerConfig) error { if s == nil || len(s.Items) == 0 { return nil } res := make(map[string]*Mquery, len(s.Items)) for _, query := range s.Items { - query.RefreshAsFilter(ownerMRN, schema) + _, err := query.RefreshAsFilter(ownerMRN, conf) + if err != nil { + return err + } if _, ok := res[query.CodeId]; ok { continue diff --git a/explorer/impact.go b/explorer/impact.go index 92bb527d1b..d53f512a4d 100644 --- a/explorer/impact.go +++ b/explorer/impact.go @@ -11,6 +11,24 @@ import ( "gopkg.in/yaml.v3" ) +func (v *Impact) HumanReadable() string { + if v.Value == nil { + return "unknown" + } + switch { + case v.Value.Value >= 90: + return "critical" + case v.Value.Value >= 70: + return "high" + case v.Value.Value >= 40: + return "medium" + case v.Value.Value > 0: + return "low" + default: + return "info" + } +} + func (v *Impact) AddBase(base *Impact) { if base == nil { return diff --git a/explorer/mquery.go b/explorer/mquery.go index 20af780d43..b6df648a10 100644 --- a/explorer/mquery.go +++ b/explorer/mquery.go @@ -11,7 +11,6 @@ import ( "strings" "github.com/rs/zerolog/log" - "go.mondoo.com/cnquery/v9" "go.mondoo.com/cnquery/v9/checksums" llx "go.mondoo.com/cnquery/v9/llx" "go.mondoo.com/cnquery/v9/mqlc" @@ -24,7 +23,7 @@ import ( // Compile a given query and return the bundle. Both v1 and v2 versions are compiled. // Both versions will be given the same code id. -func (m *Mquery) Compile(props map[string]*llx.Primitive, schema llx.Schema) (*llx.CodeBundle, error) { +func (m *Mquery) Compile(props map[string]*llx.Primitive, conf mqlc.CompilerConfig) (*llx.CodeBundle, error) { if m.Mql == "" { if m.Query == "" { return nil, errors.New("query is not implemented '" + m.Mrn + "'") @@ -33,7 +32,7 @@ func (m *Mquery) Compile(props map[string]*llx.Primitive, schema llx.Schema) (*l m.Query = "" } - v2Code, err := mqlc.Compile(m.Mql, props, mqlc.NewConfig(schema, cnquery.DefaultFeatures)) + v2Code, err := mqlc.Compile(m.Mql, props, conf) if err != nil { return nil, err } @@ -111,7 +110,7 @@ func (m *ObjectRef) RefreshMRN(ownerMRN string) error { // since their internal checksum is not stored in this query. func (m *Mquery) RefreshChecksum( ctx context.Context, - schema llx.Schema, + conf mqlc.CompilerConfig, getQuery func(ctx context.Context, mrn string) (*Mquery, error), ) error { c := checksums.New. @@ -125,7 +124,7 @@ func (m *Mquery) RefreshChecksum( for i := range m.Props { prop := m.Props[i] - if _, err := prop.RefreshChecksumAndType(schema); err != nil { + if _, err := prop.RefreshChecksumAndType(conf); err != nil { return err } if prop.Checksum == "" { @@ -137,7 +136,7 @@ func (m *Mquery) RefreshChecksum( for i := range m.Variants { ref := m.Variants[i] if q, err := getQuery(context.Background(), ref.Mrn); err == nil { - if err := q.RefreshChecksum(ctx, schema, getQuery); err != nil { + if err := q.RefreshChecksum(ctx, conf, getQuery); err != nil { return err } if q.Checksum == "" { @@ -160,7 +159,10 @@ func (m *Mquery) RefreshChecksum( Str("mql", m.Mql). Str("filter", query.Mql). Msg("refresh checksum on filter of query , which should have been pre-compiled") - query.RefreshAsFilter(m.Mrn, schema) + _, err := query.RefreshAsFilter(m.Mrn, conf) + if err != nil { + return multierr.Wrap(err, "cannot refresh checksum for query, failed to compile") + } if query.Checksum == "" { return errors.New("cannot refresh checksum for query, its filters were not compiled") } @@ -200,8 +202,8 @@ func (m *Mquery) RefreshChecksum( } // RefreshChecksumAndType by compiling the query and updating the Checksum field -func (m *Mquery) RefreshChecksumAndType(queries map[string]*Mquery, props map[string]PropertyRef, schema llx.Schema) (*llx.CodeBundle, error) { - return m.refreshChecksumAndType(queries, props, schema) +func (m *Mquery) RefreshChecksumAndType(queries map[string]*Mquery, props map[string]PropertyRef, conf mqlc.CompilerConfig) (*llx.CodeBundle, error) { + return m.refreshChecksumAndType(queries, props, conf) } type QueryMap map[string]*Mquery @@ -218,7 +220,7 @@ func (m QueryMap) GetQuery(ctx context.Context, mrn string) (*Mquery, error) { return res, nil } -func (m *Mquery) refreshChecksumAndType(queries map[string]*Mquery, props map[string]PropertyRef, schema llx.Schema) (*llx.CodeBundle, error) { +func (m *Mquery) refreshChecksumAndType(queries map[string]*Mquery, props map[string]PropertyRef, conf mqlc.CompilerConfig) (*llx.CodeBundle, error) { localProps := map[string]*llx.Primitive{} for i := range m.Props { prop := m.Props[i] @@ -246,10 +248,10 @@ func (m *Mquery) refreshChecksumAndType(queries map[string]*Mquery, props map[st if m.Mql != "" { log.Warn().Str("msn", m.Mrn).Msg("a composed query is trying to define an mql snippet, which will be ignored") } - return nil, m.RefreshChecksum(context.Background(), schema, QueryMap(queries).GetQuery) + return nil, m.RefreshChecksum(context.Background(), conf, QueryMap(queries).GetQuery) } - bundle, err := m.Compile(localProps, schema) + bundle, err := m.Compile(localProps, conf) if err != nil { return bundle, multierr.Wrap(err, "failed to compile query '"+m.Mql+"'") } @@ -275,12 +277,12 @@ func (m *Mquery) refreshChecksumAndType(queries map[string]*Mquery, props map[st m.Type = string(types.Any) } - return bundle, m.RefreshChecksum(context.Background(), schema, QueryMap(queries).GetQuery) + return bundle, m.RefreshChecksum(context.Background(), conf, QueryMap(queries).GetQuery) } // RefreshAsFilter filters treats this query as an asset filter and sets its Mrn, Title, and Checksum -func (m *Mquery) RefreshAsFilter(mrn string, schema llx.Schema) (*llx.CodeBundle, error) { - bundle, err := m.refreshChecksumAndType(nil, nil, schema) +func (m *Mquery) RefreshAsFilter(mrn string, conf mqlc.CompilerConfig) (*llx.CodeBundle, error) { + bundle, err := m.refreshChecksumAndType(nil, nil, conf) if err != nil { return bundle, err } @@ -295,7 +297,7 @@ func (m *Mquery) RefreshAsFilter(mrn string, schema llx.Schema) (*llx.CodeBundle } if checksumInvalidated { - if err := m.RefreshChecksum(context.Background(), schema, nil); err != nil { + if err := m.RefreshChecksum(context.Background(), conf, nil); err != nil { return nil, err } } @@ -468,9 +470,9 @@ func (r *Remediation) MarshalJSON() ([]byte, error) { return json.Marshal(r.Items) } -func ChecksumFilters(queries []*Mquery, schema llx.Schema) (string, error) { +func ChecksumFilters(queries []*Mquery, conf mqlc.CompilerConfig) (string, error) { for i := range queries { - if _, err := queries[i].refreshChecksumAndType(nil, nil, schema); err != nil { + if _, err := queries[i].refreshChecksumAndType(nil, nil, conf); err != nil { return "", multierr.Wrap(err, "failed to compile query") } } diff --git a/explorer/mquery_test.go b/explorer/mquery_test.go index 6abd696145..04a1a43347 100644 --- a/explorer/mquery_test.go +++ b/explorer/mquery_test.go @@ -10,6 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.mondoo.com/cnquery/v9" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/providers-sdk/v1/testutils" ) @@ -20,13 +22,14 @@ func TestMquery_RefreshAsAssetFilterStableChecksum(t *testing.T) { } x := testutils.LinuxMock() + conf := mqlc.NewConfig(x.Schema(), cnquery.DefaultFeatures) - _, err := m.RefreshAsFilter("//owner/me", x.Schema()) + _, err := m.RefreshAsFilter("//owner/me", conf) require.NoError(t, err) assert.Equal(t, "//owner/me/filter/"+m.CodeId, m.Mrn) cs := m.Checksum - _, err = m.RefreshAsFilter("//owner/me", x.Schema()) + _, err = m.RefreshAsFilter("//owner/me", conf) require.NoError(t, err) assert.Equal(t, cs, m.Checksum) } @@ -46,9 +49,10 @@ func TestMquery_Refresh(t *testing.T) { assert.Empty(t, a.Props[0].Uid) x := testutils.LinuxMock() + conf := mqlc.NewConfig(x.Schema(), cnquery.DefaultFeatures) err = a.RefreshChecksum( context.Background(), - x.Schema(), + conf, func(ctx context.Context, mrn string) (*Mquery, error) { return nil, nil }, diff --git a/explorer/property.go b/explorer/property.go index faecc8458a..4f4e477d7d 100644 --- a/explorer/property.go +++ b/explorer/property.go @@ -7,7 +7,6 @@ import ( "errors" "github.com/rs/zerolog/log" - "go.mondoo.com/cnquery/v9" "go.mondoo.com/cnquery/v9/checksums" llx "go.mondoo.com/cnquery/v9/llx" "go.mondoo.com/cnquery/v9/mqlc" @@ -43,17 +42,17 @@ func (p *Property) RefreshMRN(ownerMRN string) error { } // Compile a given property and return the bundle. -func (p *Property) Compile(props map[string]*llx.Primitive, schema llx.Schema) (*llx.CodeBundle, error) { - return mqlc.Compile(p.Mql, props, mqlc.NewConfig(schema, cnquery.DefaultFeatures)) +func (p *Property) Compile(props map[string]*llx.Primitive, conf mqlc.CompilerConfig) (*llx.CodeBundle, error) { + return mqlc.Compile(p.Mql, props, conf) } // RefreshChecksumAndType by compiling the query and updating the Checksum field -func (p *Property) RefreshChecksumAndType(schema llx.Schema) (*llx.CodeBundle, error) { - return p.refreshChecksumAndType(schema) +func (p *Property) RefreshChecksumAndType(conf mqlc.CompilerConfig) (*llx.CodeBundle, error) { + return p.refreshChecksumAndType(conf) } -func (p *Property) refreshChecksumAndType(schema llx.Schema) (*llx.CodeBundle, error) { - bundle, err := p.Compile(nil, schema) +func (p *Property) refreshChecksumAndType(conf mqlc.CompilerConfig) (*llx.CodeBundle, error) { + bundle, err := p.Compile(nil, conf) if err != nil { return bundle, multierr.Wrap(err, "failed to compile property '"+p.Mql+"'") } diff --git a/explorer/query_conductor.go b/explorer/query_conductor.go index 9f0abb9126..6829d7f736 100644 --- a/explorer/query_conductor.go +++ b/explorer/query_conductor.go @@ -10,7 +10,9 @@ import ( "strings" "github.com/rs/zerolog/log" + "go.mondoo.com/cnquery/v9" llx "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/mrn" "go.mondoo.com/cnquery/v9/utils/multierr" "go.mondoo.com/ranger-rpc/codes" @@ -92,7 +94,8 @@ func (s *LocalServices) SetProps(ctx context.Context, req *PropsReq) (*Empty, er // validate that the queries compile and fill in checksums for i := range req.Props { prop := req.Props[i] - code, err := prop.RefreshChecksumAndType(s.runtime.Schema()) + conf := mqlc.NewConfig(s.runtime.Schema(), cnquery.DefaultFeatures) + code, err := prop.RefreshChecksumAndType(conf) if err != nil { return nil, err } @@ -192,6 +195,8 @@ func (s *LocalServices) addQueryToJob(ctx context.Context, query *Mquery, job *E return nil } + compilerConfig := mqlc.NewConfig(s.runtime.Schema(), cnquery.DefaultFeatures) + var props map[string]*llx.Primitive var propRefs map[string]string if len(query.Props) != 0 { @@ -220,7 +225,7 @@ func (s *LocalServices) addQueryToJob(ctx context.Context, query *Mquery, job *E continue } - code, err := prop.Compile(nil, s.runtime.Schema()) + code, err := prop.Compile(nil, compilerConfig) if err != nil { return multierr.Wrap(err, "failed to compile property for query "+query.Mrn) } @@ -243,7 +248,7 @@ func (s *LocalServices) addQueryToJob(ctx context.Context, query *Mquery, job *E return nil } - codeBundle, err := query.Compile(props, s.runtime.Schema()) + codeBundle, err := query.Compile(props, compilerConfig) if err != nil { return err } @@ -303,7 +308,8 @@ func MatchFilters(entityMrn string, filters []*Mquery, packs []*QueryPack, schem return "", NewAssetMatchError(entityMrn, "querypacks", "no-matching-packs", filters, &Filters{Items: supported}) } - sum, err := ChecksumFilters(matching, schema) + conf := mqlc.NewConfig(schema, cnquery.DefaultFeatures) + sum, err := ChecksumFilters(matching, conf) if err != nil { return "", err } diff --git a/explorer/query_conductor_test.go b/explorer/query_conductor_test.go index 8975dc7de7..2e5bf4c598 100644 --- a/explorer/query_conductor_test.go +++ b/explorer/query_conductor_test.go @@ -8,18 +8,23 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.mondoo.com/cnquery/v9" + "go.mondoo.com/cnquery/v9/mqlc" "go.mondoo.com/cnquery/v9/providers" ) func TestMatchFilters(t *testing.T) { schema := providers.DefaultRuntime().Schema() + conf := mqlc.NewConfig(schema, cnquery.DefaultFeatures) t.Run("one matching filter", func(t *testing.T) { filters := NewFilters("true", "false") - filters.Compile("//owner", schema) + err := filters.Compile("//owner", conf) + require.NoError(t, err) matching := []*Mquery{{Mql: "true"}} - ChecksumFilters(matching, schema) + _, err = ChecksumFilters(matching, conf) + require.NoError(t, err) res, err := MatchFilters("assetMrn", matching, []*QueryPack{{ComputedFilters: filters}}, schema) require.NoError(t, err) @@ -28,12 +33,14 @@ func TestMatchFilters(t *testing.T) { t.Run("no matching filter (matching is provided)", func(t *testing.T) { filters := NewFilters("true", "false") - filters.Compile("//owner", schema) + err := filters.Compile("//owner", conf) + require.NoError(t, err) matching := []*Mquery{{Mql: "0"}} - ChecksumFilters(matching, schema) + _, err = ChecksumFilters(matching, conf) + require.NoError(t, err) - _, err := MatchFilters("assetMrn", matching, []*QueryPack{{ComputedFilters: filters}}, schema) + _, err = MatchFilters("assetMrn", matching, []*QueryPack{{ComputedFilters: filters}}, schema) assert.EqualError(t, err, "rpc error: code = InvalidArgument desc = asset isn't supported by any querypacks\n"+ "querypacks support: false, true\n"+ @@ -42,9 +49,10 @@ func TestMatchFilters(t *testing.T) { t.Run("no matching filter (matching is empty)", func(t *testing.T) { filters := NewFilters("true", "false") - filters.Compile("//owner", schema) + err := filters.Compile("//owner", conf) + require.NoError(t, err) - _, err := MatchFilters("assetMrn", []*Mquery{}, []*QueryPack{{ComputedFilters: filters}}, schema) + _, err = MatchFilters("assetMrn", []*Mquery{}, []*QueryPack{{ComputedFilters: filters}}, schema) assert.EqualError(t, err, "rpc error: code = InvalidArgument desc = asset doesn't support any querypacks") }) diff --git a/mqlc/mqlc.go b/mqlc/mqlc.go index 3f26046fdb..6d018bab71 100644 --- a/mqlc/mqlc.go +++ b/mqlc/mqlc.go @@ -86,20 +86,22 @@ func (vm *varmap) len() int { return len(vm.vars) } -type compilerConfig struct { +type CompilerConfig struct { Schema llx.Schema UseAssetContext bool Stats CompilerStats } -func (c *compilerConfig) EnableStats() { - c.Stats = &compilerStats{ - ResourceFields: map[string]map[string]FieldStat{}, - } +func (c *CompilerConfig) EnableStats() { + c.Stats = newCompilerStats() +} + +func (c *CompilerConfig) EnableMultiStats() { + c.Stats = newCompilerMultiStats() } -func NewConfig(schema llx.Schema, features cnquery.Features) compilerConfig { - return compilerConfig{ +func NewConfig(schema llx.Schema, features cnquery.Features) CompilerConfig { + return CompilerConfig{ Schema: schema, UseAssetContext: features.IsActive(cnquery.MQLAssetContext), Stats: compilerStatsNull{}, @@ -107,7 +109,7 @@ func NewConfig(schema llx.Schema, features cnquery.Features) compilerConfig { } type compiler struct { - compilerConfig + CompilerConfig Result *llx.CodeBundle Binding *variable @@ -163,7 +165,7 @@ func (c *compiler) newBlockCompiler(binding *variable) compiler { } return compiler{ - compilerConfig: c.compilerConfig, + CompilerConfig: c.CompilerConfig, Result: c.Result, Binding: binding, blockDeps: blockDeps, @@ -984,7 +986,7 @@ func (c *compiler) compileBoundIdentifierWithMqlCtx(id string, binding *variable fieldPath, fieldinfos, ok := c.findField(resource, id) if ok { fieldinfo := fieldinfos[len(fieldinfos)-1] - c.compilerConfig.Stats.CallField(resource.Name, fieldinfo) + c.CompilerConfig.Stats.CallField(resource.Name, fieldinfo) if call != nil && len(call.Function) > 0 && !fieldinfo.IsImplicitResource { return true, types.Nil, errors.New("cannot call resource field with arguments yet") @@ -1078,7 +1080,7 @@ func (c *compiler) compileBoundIdentifierWithoutMqlCtx(id string, binding *varia } if fieldinfo != nil { - c.compilerConfig.Stats.CallField(resource.Name, fieldinfo) + c.CompilerConfig.Stats.CallField(resource.Name, fieldinfo) if call != nil && len(call.Function) > 0 { return true, types.Nil, errors.New("cannot call resource field with arguments yet") @@ -1148,7 +1150,7 @@ func (c *compiler) compileResource(id string, calls []*parser.Call) (bool, []*pa calls = calls[1:] } - c.compilerConfig.Stats.CallResource(resource.Name) + c.CompilerConfig.Stats.CallResource(resource.Name) var call *parser.Call if len(calls) > 0 && calls[0].Function != nil { @@ -2195,7 +2197,7 @@ func getMinMondooVersion(schema llx.Schema, current string, resource string, fie } // CompileAST with a schema into a chunky code -func CompileAST(ast *parser.AST, props map[string]*llx.Primitive, conf compilerConfig) (*llx.CodeBundle, error) { +func CompileAST(ast *parser.AST, props map[string]*llx.Primitive, conf CompilerConfig) (*llx.CodeBundle, error) { if conf.Schema == nil { return nil, errors.New("mqlc> please provide a schema to compile this code") } @@ -2221,7 +2223,7 @@ func CompileAST(ast *parser.AST, props map[string]*llx.Primitive, conf compilerC } c := compiler{ - compilerConfig: conf, + CompilerConfig: conf, Result: codeBundle, vars: newvarmap(1<<32, nil), parent: nil, @@ -2235,7 +2237,11 @@ func CompileAST(ast *parser.AST, props map[string]*llx.Primitive, conf compilerC } // Compile a code piece against a schema into chunky code -func compile(input string, props map[string]*llx.Primitive, conf compilerConfig) (*llx.CodeBundle, error) { +func compile(input string, props map[string]*llx.Primitive, compilerConf CompilerConfig) (*llx.CodeBundle, error) { + + conf := compilerConf + conf.Stats = compilerConf.Stats.CompileQuery(input) + // remove leading whitespace; we are re-using this later on input = Dedent(input) @@ -2274,7 +2280,7 @@ func compile(input string, props map[string]*llx.Primitive, conf compilerConfig) return res, nil } -func Compile(input string, props map[string]*llx.Primitive, conf compilerConfig) (*llx.CodeBundle, error) { +func Compile(input string, props map[string]*llx.Primitive, conf CompilerConfig) (*llx.CodeBundle, error) { // Note: we do not check the conf because it will get checked by the // first CompileAST call. Do not use it earlier or add a check. diff --git a/mqlc/mqlc_stats.go b/mqlc/mqlc_stats.go index dd009b4663..1c014d954a 100644 --- a/mqlc/mqlc_stats.go +++ b/mqlc/mqlc_stats.go @@ -5,17 +5,24 @@ package mqlc import ( "sort" + "sync" "time" + "github.com/rs/zerolog/log" "go.mondoo.com/cnquery/v9/providers-sdk/v1/resources" "go.mondoo.com/cnquery/v9/types" ) type CompilerStats interface { WalkSorted(f func(resource string, field string, info FieldStat)) + WalkCode(f func(code string, stats CompilerStats)) + + // calls used by mqlc compiler internally: SetAutoExpand(on bool) CallResource(name string) CallField(resource string, field *resources.Field) + // returns an object to track the compilation of a specific query + CompileQuery(query string) CompilerStats } type FieldStat struct { @@ -26,10 +33,15 @@ type FieldStat struct { type compilerStatsNull struct{} +// interface validation +var _ CompilerStats = compilerStatsNull{} + func (c compilerStatsNull) WalkSorted(f func(resource string, field string, info FieldStat)) {} +func (c compilerStatsNull) WalkCode(f func(code string, stats CompilerStats)) {} func (c compilerStatsNull) SetAutoExpand(on bool) {} func (c compilerStatsNull) CallResource(name string) {} func (c compilerStatsNull) CallField(resource string, field *resources.Field) {} +func (c compilerStatsNull) CompileQuery(query string) CompilerStats { return c } type compilerStats struct { ResourceFields map[string]map[string]FieldStat @@ -39,6 +51,15 @@ type compilerStats struct { isAutoExpand bool } +// interface validation +var _ CompilerStats = &compilerStats{} + +func newCompilerStats() *compilerStats { + return &compilerStats{ + ResourceFields: map[string]map[string]FieldStat{}, + } +} + func (c *compilerStats) SetAutoExpand(on bool) { if c == nil { return @@ -68,6 +89,8 @@ func (c *compilerStats) CallField(resource string, field *resources.Field) { } } +func (c *compilerStats) CompileQuery(query string) CompilerStats { return c } + func (c *compilerStats) WalkSorted(f func(resource string, field string, info FieldStat)) { sortedResources := make([]string, len(c.ResourceFields)) i := 0 @@ -98,3 +121,64 @@ func (c *compilerStats) WalkSorted(f func(resource string, field string, info Fi } } } + +func (c *compilerStats) WalkCode(f func(code string, stats CompilerStats)) {} + +// interface validation +var _ CompilerStats = &compilerMultiStats{} + +type compilerMultiStats struct { + stats map[string]*compilerStats + lock sync.Mutex + + AutoExpand bool +} + +func newCompilerMultiStats() *compilerMultiStats { + return &compilerMultiStats{ + stats: map[string]*compilerStats{}, + } +} + +func (c *compilerMultiStats) WalkSorted(f func(resource string, field string, info FieldStat)) { + panic("walking code on multi-stats not yet supported") +} + +func (c *compilerMultiStats) WalkCode(f func(code string, stats CompilerStats)) { + c.lock.Lock() + defer c.lock.Unlock() + + for k, v := range c.stats { + f(k, v) + } +} + +// The errors currently are soft only. I'm not sure if we shouldn't switch +// them to be much stricter, because this is something that should never +// happen and points to bad coding errors. + +func (c *compilerMultiStats) SetAutoExpand(on bool) { + log.Error().Msg("using uninitialized compiler multi-stats, internal error") +} + +func (c *compilerMultiStats) CallResource(name string) { + log.Error().Msg("using uninitialized compiler multi-stats, internal error") +} + +func (c *compilerMultiStats) CallField(resource string, field *resources.Field) { + log.Error().Msg("using uninitialized compiler multi-stats, internal error") +} + +func (c *compilerMultiStats) CompileQuery(query string) CompilerStats { + c.lock.Lock() + defer c.lock.Unlock() + + existing, ok := c.stats[query] + if ok { + return existing + } + + res := newCompilerStats() + c.stats[query] = res + return res +}