⭐️ Allow loading policies from an s3 bucket. (#988)

* ⭐️ Allow loading policies from an s3 bucket.

Signed-off-by: Preslav <[email protected]>

* Add s3 tests with an s3 fake. Update example policies to v8 format.

---------

Signed-off-by: Preslav <[email protected]>

apps/cnspec/cmd/bundle.go

@@ -204,7 +204,8 @@ var policyPublishCmd = &cobra.Command{
 		}
 
 		// compile manipulates the bundle, therefore we read it again
-		policyBundle, err := policy.BundleFromPaths(filename)
+		bundleLoader := policy.DefaultBundleLoader()
+		policyBundle, err := bundleLoader.BundleFromPaths(filename)
 		if err != nil {
 			log.Fatal().Err(err).Msg("could not load policy bundle")
 		}
@@ -271,7 +272,8 @@ var policyDocsCmd = &cobra.Command{
 		viper.BindPFlag("no-code", cmd.Flags().Lookup("no-code"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		bundle, err := policy.BundleFromPaths(args...)
+		bundleLoader := policy.DefaultBundleLoader()
+		bundle, err := bundleLoader.BundleFromPaths(args...)
 		if err != nil {
 			log.Fatal().Err(err).Msg("failed to load bundle")
 		}

apps/cnspec/cmd/scan.go

@@ -319,7 +319,8 @@ func (c *scanConfig) loadPolicies() error {
 			return nil
 		}
 
-		bundle, err := policy.BundleFromPaths(c.PolicyPaths...)
+		bundleLoader := policy.DefaultBundleLoader()
+		bundle, err := bundleLoader.BundleFromPaths(c.PolicyPaths...)
 		if err != nil {
 			return err
 		}

examples/directory/example1.mql.yaml

@@ -8,12 +8,11 @@ policies:
     authors:
       - name: Mondoo
         email: [email protected]
-    specs:
-      - asset_filter:
-          query: platform.family.contains(_ == 'unix')
-        scoring_queries:
-          sshd-01: {}
-          sshd-02: {}
-          sshd-03: {}
-        data_queries:
-          sshd-d-1: 0
+    groups:
+      - filters: platform.family.contains('unix')
+        checks:
+          - uid: sshd-01
+          - uid: sshd-02
+          - uid: sshd-03
+        queries:
+          - uid: sshd-d-1

examples/directory/example2.mql.yaml

@@ -5,10 +5,9 @@ policies:
   - uid: example2
     name: Another policy
     version: 1.0.0
-    specs:
-      - asset_filter:
-          query: platform.family.contains(_ == 'unix')
+    groups:
+      - filters: platform.family.contains('unix')
+        checks:
+          - uid: linux-1
         policies:
-          example1: {}
-        scoring_queries:
-          linux-1: {}
+          - uid: example1

go.mod

@@ -10,6 +10,7 @@ require go.mondoo.com/cnquery/v9 v9.10.0
 
 require (
 	github.com/Masterminds/semver v1.5.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3
 	github.com/cockroachdb/errors v1.11.1
 	github.com/google/go-cmdtest v0.4.0
 	github.com/google/uuid v1.4.0
@@ -56,9 +57,13 @@ require (
 	github.com/alecthomas/go-check-sumtype v0.1.3 // indirect
 	github.com/alexkohler/nakedret/v2 v2.0.2 // indirect
 	github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ecr v1.24.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.21.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 // indirect
 	github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20231121224113-b6714ac5eb13 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
@@ -144,7 +149,7 @@ require (
 	github.com/atotto/clipboard v0.1.4 // indirect
 	github.com/aws/aws-sdk-go v1.48.12 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.23.5 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.25.11 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.25.11
 	github.com/aws/aws-sdk-go-v2/credentials v1.16.9 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect

go.sum

@@ -146,6 +146,8 @@ github.com/aws/aws-sdk-go v1.48.12 h1:n+eGzflzzvYubu2cOjqpVll7lF+Ci0ThyCpg5kzfzb
 github.com/aws/aws-sdk-go v1.48.12/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go-v2 v1.23.5 h1:xK6C4udTyDMd82RFvNkDQxtAd00xlzFUtX4fF2nMZyg=
 github.com/aws/aws-sdk-go-v2 v1.23.5/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 h1:Zx9+31KyB8wQna6SXFWOewlgoY5uGdDAu6PTOEU3OQI=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3/go.mod h1:zxbEJhRdKTH1nqS2qu6UJ7zGe25xaHxZXaC2CvuQFnA=
 github.com/aws/aws-sdk-go-v2/config v1.25.11 h1:RWzp7jhPRliIcACefGkKp03L0Yofmd2p8M25kbiyvno=
 github.com/aws/aws-sdk-go-v2/config v1.25.11/go.mod h1:BVUs0chMdygHsQtvaMyEOpW2GIW+ubrxJLgIz/JU29s=
 github.com/aws/aws-sdk-go-v2/credentials v1.16.9 h1:LQo3MUIOzod9JdUK+wxmSdgzLVYUbII3jXn3S/HJZU0=
@@ -158,6 +160,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 h1:abKT+RuM1sdCNZIGIfZpLkvxEX3Rpsto019XG/rkYG8=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8/go.mod h1:Owc4ysUE71JSruVTTa3h4f2pp3E4hlcAtmeNXxDmjj4=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.138.2 h1:e3Imv1oXz+W3Tfclflkh72t5TUPUwWdkHP7ctQGk8Dc=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.138.2/go.mod h1:d1hAqgLDOPaSO1Piy/0bBmj6oAplFwv6p0cquHntNHM=
 github.com/aws/aws-sdk-go-v2/service/ec2instanceconnect v1.20.2 h1:owl9n1S8bJ4w/GpXvvwC9a/zpJnfvoOyvFWHKdSPJW8=
@@ -168,8 +172,14 @@ github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.21.2 h1:BwF80uV9Ga4yB6UMUggQ4R
 github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.21.2/go.mod h1:IQcO27/ICcX5d2gBgh6sYHZ4hkcVn3n+0A6brZwOnWU=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 h1:xyfOAYV/ujzZOo01H9+OnyeiRKmTEp6EsITTsmq332Q=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8/go.mod h1:coLeQEoKzW9ViTL2bn0YUlU7K0RYjivKudG74gtd+sI=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 h1:EamsKe+ZjkOQjDdHd86/JCEucjFKQ9T0atWKO4s2Lgs=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8/go.mod h1:Q0vV3/csTpbkfKLI5Sb56cJQTCTtJ0ixdb7P+Wedqiw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 h1:ip5ia3JOXl4OAsqeTdrOOmqKgoWiu+t9XSOnRzBwmRs=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8/go.mod h1:kE+aERnK9VQIw1vrk7ElAvhCsgLNzGyCPNg2Qe4Eq4c=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3 h1:j34+Cw6EzOZmk1V505oZimpNSco1e83K7HPQKxCc0wY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3/go.mod h1:thjZng67jGsvMyVZnSxlcqKyLwB0XTG8bHIRZPTJ+Bs=
 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.2 h1:JKbfiLwEqJp8zaOAOn6AVSMS96gdwP3TjBMvZYsbxqE=
 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.2/go.mod h1:pbBOMK8UicdDK11zsPSGbpFh9Xwbd1oD3t7pSxXgNxU=
 github.com/aws/aws-sdk-go-v2/service/ssm v1.44.2 h1:lmdmYCvG1EJKGLEsUsYDNO6MwZyBZROrRg04Vrb5TwA=

internal/bundle/lint.go

@@ -171,7 +171,8 @@ func Lint(schema llx.Schema, files ...string) (*Results, error) {
 
 	// Note: we only run compile on the aggregated level to ensure the bundle in combination is valid
 	// Invalid yaml files are already caught by the individual linting, therefore we do not need extra error handling here
-	policyBundle, err := policy.BundleFromPaths(files...)
+	bundleLoader := policy.DefaultBundleLoader()
+	policyBundle, err := bundleLoader.BundleFromPaths(files...)
 	if err == nil {
 		_, err = policyBundle.Compile(context.Background(), schema, nil)
 		if err != nil {

policy/bundle.go

@@ -7,9 +7,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/fs"
-	"os"
-	"path/filepath"
 	"regexp"
 	"sort"
 	"strings"
@@ -34,106 +31,71 @@ const (
 	MRN_RESOURCE_CONTROL      = "controls"
 )
 
-// BundleExecutionChecksum creates a combined execution checksum from a policy
-// and framework. Either may be nil.
-func BundleExecutionChecksum(policy *Policy, framework *Framework) string {
-	res := checksums.New
-	if policy != nil {
-		res = res.Add(policy.GraphExecutionChecksum)
-	}
-	if framework != nil {
-		res = res.Add(framework.GraphExecutionChecksum)
-	}
-	return res.String()
+type BundleResolver interface {
+	Load(ctx context.Context, path string) (*Bundle, error)
+	IsApplicable(path string) bool
 }
 
-// BundleFromPaths loads a single policy bundle file or a bundle that
-// was split into multiple files into a single PolicyBundle struct
-func BundleFromPaths(paths ...string) (*Bundle, error) {
-	// load all the source files
-	resolvedFilenames, err := WalkPolicyBundleFiles(paths...)
-	if err != nil {
-		log.Error().Err(err).Msg("could not resolve bundle files")
-		return nil, err
-	}
-
-	// aggregate all files into a single policy bundle
-	aggregatedBundle, err := aggregateFilesToBundle(resolvedFilenames)
-	if err != nil {
-		log.Debug().Err(err).Msg("could merge bundle files")
-		return nil, err
-	}
-
-	logger.DebugDumpYAML("resolved_mql_bundle.mql", aggregatedBundle)
-	return aggregatedBundle, nil
+type BundleLoader struct {
+	resolvers []BundleResolver
 }
 
-// WalkPolicyBundleFiles iterates over all provided filenames and
-// checks if the name is a file or a directory. If the filename
-// is a directory, it walks the directory recursively
-func WalkPolicyBundleFiles(filenames ...string) ([]string, error) {
-	// resolve file names
-	resolvedFilenames := []string{}
-	for i := range filenames {
-		filename := filenames[i]
-		fi, err := os.Stat(filename)
-		if err != nil {
-			return nil, errors.Wrap(err, "could not load policy bundle file: "+filename)
-		}
-
-		if fi.IsDir() {
-			filepath.WalkDir(filename, func(path string, d fs.DirEntry, err error) error {
-				if err != nil {
-					return err
-				}
-				// we ignore directories because WalkDir already walks them
-				if d.IsDir() {
-					return nil
-				}
+func NewBundleLoader(resolvers ...BundleResolver) *BundleLoader {
+	return &BundleLoader{resolvers: resolvers}
+}
 
-				// only consider .yaml|.yml files
-				if strings.HasSuffix(d.Name(), ".mql.yaml") || strings.HasSuffix(d.Name(), ".mql.yml") {
-					resolvedFilenames = append(resolvedFilenames, path)
-				}
+func DefaultBundleLoader() *BundleLoader {
+	return NewBundleLoader(defaultS3BundleResolver(), defaultFileBundleResolver())
+}
 
-				return nil
-			})
-		} else {
-			resolvedFilenames = append(resolvedFilenames, filename)
+func (l *BundleLoader) getResolver(path string) (BundleResolver, error) {
+	for _, resolver := range l.resolvers {
+		if resolver.IsApplicable(path) {
+			return resolver, nil
 		}
 	}
-
-	return resolvedFilenames, nil
+	return nil, fmt.Errorf("no resolver found for path '%s'", path)
 }
 
-// aggregateFilesToBundle iterates over all provided files and loads its content.
-// It assumes that all provided files are checked upfront and are not a directory
-func aggregateFilesToBundle(paths []string) (*Bundle, error) {
-	// iterate over all files, load them and merge them
-	mergedBundle := &Bundle{}
+// Deprecated: Use BundleLoader.BundleFromPaths instead
+func BundleFromPaths(paths ...string) (*Bundle, error) {
+	defaultLoader := DefaultBundleLoader()
+	return defaultLoader.BundleFromPaths(paths...)
+}
 
-	for i := range paths {
-		path := paths[i]
-		log.Debug().Str("path", path).Msg("loading policy bundle file")
-		bundle, err := bundleFromSingleFile(path)
+// iterates through all the resolvers until it finds an applicable one and then uses that to load the bundle
+// from the provided path
+func (l *BundleLoader) BundleFromPaths(paths ...string) (*Bundle, error) {
+	ctx := context.Background()
+	aggregatedBundle := &Bundle{}
+	for _, path := range paths {
+		resolver, err := l.getResolver(path)
 		if err != nil {
-			return nil, errors.Wrap(err, "could not load file: "+path)
+			return nil, err
 		}
-
-		mergedBundle = Merge(mergedBundle, bundle)
+		bundle, err := resolver.Load(ctx, path)
+		if err != nil {
+			log.Error().Err(err).Msg("could not resolve bundle files")
+			return nil, err
+		}
+		aggregatedBundle = Merge(aggregatedBundle, bundle)
 	}
 
-	return mergedBundle, nil
+	logger.DebugDumpYAML("resolved_mql_bundle.mql", aggregatedBundle)
+	return aggregatedBundle, nil
 }
 
-// bundleFromSingleFile loads a policy bundle from a single file
-func bundleFromSingleFile(path string) (*Bundle, error) {
-	bundleData, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
+// BundleExecutionChecksum creates a combined execution checksum from a policy
+// and framework. Either may be nil.
+func BundleExecutionChecksum(policy *Policy, framework *Framework) string {
+	res := checksums.New
+	if policy != nil {
+		res = res.Add(policy.GraphExecutionChecksum)
 	}
-
-	return BundleFromYAML(bundleData)
+	if framework != nil {
+		res = res.Add(framework.GraphExecutionChecksum)
+	}
+	return res.String()
 }
 
 // Merge combines two PolicyBundle and merges the data additive into one