🐛 Fix tree nodes ordering, makes query checksums deterministic. (#1790)

mqlc/mqlc.go

@@ -1862,7 +1862,13 @@ func (c *compiler) addValueFieldChunks(ref uint64) {
 			return
 		}
 		path = append(path, node.id)
-		for _, child := range node.children {
+		keys := []string{}
+		for k := range node.children {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, k := range keys {
+			child := node.children[k]
 			visit(tree, child, path)
 			visitTreeNodes(tree, child, path, visit)
 		}

mqlc/mqlc_test.go

@@ -15,14 +15,15 @@ import (
 	"go.mondoo.com/cnquery/llx"
 	"go.mondoo.com/cnquery/logger"
 	"go.mondoo.com/cnquery/mqlc"
-	"go.mondoo.com/cnquery/providers"
 	"go.mondoo.com/cnquery/types"
+
+	"go.mondoo.com/cnquery/providers-sdk/v1/testutils"
 )
 
 var (
 	features    = cnquery.Features{}
-	os_schema   = providers.MustLoadSchemaFromFile("os", "../providers/os/resources/os.resources.json")
-	core_schema = providers.MustLoadSchemaFromFile("core", "../providers/core/resources/core.resources.json")
+	core_schema = testutils.MustLoadSchema(testutils.SchemaProvider{Provider: "core"})
+	os_schema   = testutils.MustLoadSchema(testutils.SchemaProvider{Provider: "os"})
 	conf        = mqlc.NewConfig(
 		core_schema.Add(os_schema),
 		features,
@@ -235,6 +236,20 @@ func TestCompiler_Semicolon(t *testing.T) {
 	})
 }
 
+func TestCompiler_DeterministicChecksum(t *testing.T) {
+	// this is a query that in the past used to produce different checksum every now and then
+	// this test ensures that the checksum is always deterministic now
+	mql := `azure.subscription.sql.servers.all(databases.one (transparentDataEncryption["state"] == "Enabled") && encryptionProtector["serverKeyType"] == "AzureKeyVault" )`
+	azure_schema := testutils.MustLoadSchema(testutils.SchemaProvider{Provider: "azure"})
+
+	for i := 0; i < 10_000; i++ {
+		azureConf := mqlc.NewConfig(azure_schema, features)
+		res, err := mqlc.Compile(mql, map[string]*llx.Primitive{}, azureConf)
+		require.Nil(t, err)
+		require.Equal(t, res.CodeV2.Id, "LkB8PP3xB2Q=")
+	}
+}
+
 func TestCompiler_Simple(t *testing.T) {
 	data := []struct {
 		code string

providers-sdk/v1/lr/cli/cmd/go.go

@@ -13,7 +13,6 @@ import (
 	"github.com/spf13/cobra"
 	"go.mondoo.com/cnquery/providers-sdk/v1/lr"
 	"go.mondoo.com/cnquery/providers-sdk/v1/lr/docs"
-	"go.mondoo.com/cnquery/providers-sdk/v1/resources"
 	"sigs.k8s.io/yaml"
 )
 
@@ -66,7 +65,7 @@ var goCmd = &cobra.Command{
 				log.Fatal().Err(err).Msg("could not load yaml data")
 			}
 
-			injectMetadata(schema, &lrDocsData)
+			docs.InjectMetadata(schema, &lrDocsData)
 		} else if os.IsNotExist(err) {
 			log.Info().Str("path", manifestPath).Msg("no manifest found, ignoring")
 		} else {
@@ -104,23 +103,3 @@ func init() {
 	rootCmd.AddCommand(goCmd)
 	goCmd.Flags().String("dist", "", "folder for output json generation")
 }
-
-func injectMetadata(schema *resources.Schema, docs *docs.LrDocs) {
-	for resource, rdoc := range docs.Resources {
-		info, ok := schema.Resources[resource]
-		if !ok {
-			continue
-		}
-
-		info.MinMondooVersion = rdoc.MinMondooVersion
-
-		for field, fdoc := range rdoc.Fields {
-			finfo, ok := info.Fields[field]
-			if !ok {
-				continue
-			}
-
-			finfo.MinMondooVersion = fdoc.MinMondooVersion
-		}
-	}
-}

providers-sdk/v1/lr/docs/docs.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	"go.mondoo.com/cnquery/providers-sdk/v1/resources"
 )
 
 type LrDocs struct {
@@ -74,3 +76,23 @@ type LrDocsSnippet struct {
 	Title string `json:"title,omitempty"`
 	Query string `json:"query,omitempty"`
 }
+
+func InjectMetadata(schema *resources.Schema, docs *LrDocs) {
+	for resource, rdoc := range docs.Resources {
+		info, ok := schema.Resources[resource]
+		if !ok {
+			continue
+		}
+
+		info.MinMondooVersion = rdoc.MinMondooVersion
+
+		for field, fdoc := range rdoc.Fields {
+			finfo, ok := info.Fields[field]
+			if !ok {
+				continue
+			}
+
+			finfo.MinMondooVersion = fdoc.MinMondooVersion
+		}
+	}
+}

providers-sdk/v1/testutils/testutils.go

@@ -23,13 +23,15 @@ import (
 	"go.mondoo.com/cnquery/mqlc"
 	"go.mondoo.com/cnquery/providers"
 	"go.mondoo.com/cnquery/providers-sdk/v1/lr"
+	"go.mondoo.com/cnquery/providers-sdk/v1/lr/docs"
 	"go.mondoo.com/cnquery/providers-sdk/v1/resources"
 	"go.mondoo.com/cnquery/providers-sdk/v1/testutils/mockprovider"
 	"go.mondoo.com/cnquery/providers/mock"
 	networkconf "go.mondoo.com/cnquery/providers/network/config"
 	networkprovider "go.mondoo.com/cnquery/providers/network/provider"
 	osconf "go.mondoo.com/cnquery/providers/os/config"
 	osprovider "go.mondoo.com/cnquery/providers/os/provider"
+	"sigs.k8s.io/yaml"
 )
 
 var (
@@ -81,6 +83,11 @@ type tester struct {
 	Runtime llx.Runtime
 }
 
+type SchemaProvider struct {
+	Provider string
+	Path     string
+}
+
 func InitTester(runtime llx.Runtime) *tester {
 	return &tester{
 		Runtime: runtime,
@@ -150,32 +157,57 @@ func (ctx *tester) TestMqlc(t *testing.T, bundle *llx.CodeBundle, props map[stri
 	return results
 }
 
-func mustLoadSchema(provider string) *resources.Schema {
+func MustLoadSchema(provider SchemaProvider) *resources.Schema {
+	if provider.Path == "" && provider.Provider == "" {
+		panic("cannot load schema without provider name or path")
+	}
 	var path string
-	if provider == "mockprovider" {
-		path = filepath.Join(TestutilsDir, "mockprovider/resources/mockprovider.lr")
-	} else {
-		path = filepath.Join(TestutilsDir, "../../../providers/"+provider+"/resources/"+provider+".lr")
+	// path towards the .yaml manifest, containing metadata abou the resources
+	var manifestPath string
+	if provider.Provider != "" {
+		switch provider.Provider {
+		// special handling for the mockprovider
+		case "mockprovider":
+			path = filepath.Join(TestutilsDir, "mockprovider/resources/mockprovider.lr")
+		default:
+			manifestPath = filepath.Join(TestutilsDir, "../../../providers/"+provider.Provider+"/resources/"+provider.Provider+".lr.manifest.yaml")
+			path = filepath.Join(TestutilsDir, "../../../providers/"+provider.Provider+"/resources/"+provider.Provider+".lr")
+		}
+	} else if provider.Path != "" {
+		path = provider.Path
 	}
 
 	res, err := lr.Resolve(path, func(path string) ([]byte, error) { return os.ReadFile(path) })
 	if err != nil {
 		panic(err.Error())
 	}
-
 	schema, err := lr.Schema(res)
 	if err != nil {
 		panic(err.Error())
 	}
+	// TODO: we should make a function that takes the Schema and the metadata and merges those.
+	// Then we can use that in the LR code and the testutils code too
+	if manifestPath != "" {
+		// we will attempt to auto-detect the manifest to inject some metadata
+		// into the schema
+		raw, err := os.ReadFile(manifestPath)
+		if err == nil {
+			var lrDocsData docs.LrDocs
+			err = yaml.Unmarshal(raw, &lrDocsData)
+			if err == nil {
+				docs.InjectMetadata(schema, &lrDocsData)
+			}
+		}
+	}
 
 	return schema
 }
 
 func Local() llx.Runtime {
-	osSchema := mustLoadSchema("os")
-	coreSchema := mustLoadSchema("core")
-	networkSchema := mustLoadSchema("network")
-	mockSchema := mustLoadSchema("mockprovider")
+	osSchema := MustLoadSchema(SchemaProvider{Provider: "os"})
+	coreSchema := MustLoadSchema(SchemaProvider{Provider: "core"})
+	networkSchema := MustLoadSchema(SchemaProvider{Provider: "network"})
+	mockSchema := MustLoadSchema(SchemaProvider{Provider: "mockprovider"})
 
 	runtime := providers.Coordinator.NewRuntime()