diff --git a/go.mod b/go.mod index aceca6a3f..ab5c78607 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require go.mondoo.com/cnquery/v9 v9.10.0 require ( github.com/Masterminds/semver v1.5.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3 github.com/cockroachdb/errors v1.11.1 github.com/google/go-cmdtest v0.4.0 github.com/google/uuid v1.4.0 @@ -56,9 +57,13 @@ require ( github.com/alecthomas/go-check-sumtype v0.1.3 // indirect github.com/alexkohler/nakedret/v2 v2.0.2 // indirect github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 // indirect github.com/aws/aws-sdk-go-v2/service/ecr v1.24.2 // indirect github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.21.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 // indirect github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20231121224113-b6714ac5eb13 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect @@ -144,7 +149,7 @@ require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go v1.48.12 // indirect github.com/aws/aws-sdk-go-v2 v1.23.5 // indirect - github.com/aws/aws-sdk-go-v2/config v1.25.11 // indirect + github.com/aws/aws-sdk-go-v2/config v1.25.11 github.com/aws/aws-sdk-go-v2/credentials v1.16.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect diff --git a/go.sum b/go.sum index 88997e664..0a1f11c2b 100644 --- a/go.sum +++ b/go.sum @@ -146,6 +146,8 @@ github.com/aws/aws-sdk-go v1.48.12 h1:n+eGzflzzvYubu2cOjqpVll7lF+Ci0ThyCpg5kzfzb github.com/aws/aws-sdk-go v1.48.12/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.23.5 h1:xK6C4udTyDMd82RFvNkDQxtAd00xlzFUtX4fF2nMZyg= github.com/aws/aws-sdk-go-v2 v1.23.5/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 h1:Zx9+31KyB8wQna6SXFWOewlgoY5uGdDAu6PTOEU3OQI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3/go.mod h1:zxbEJhRdKTH1nqS2qu6UJ7zGe25xaHxZXaC2CvuQFnA= github.com/aws/aws-sdk-go-v2/config v1.25.11 h1:RWzp7jhPRliIcACefGkKp03L0Yofmd2p8M25kbiyvno= github.com/aws/aws-sdk-go-v2/config v1.25.11/go.mod h1:BVUs0chMdygHsQtvaMyEOpW2GIW+ubrxJLgIz/JU29s= github.com/aws/aws-sdk-go-v2/credentials v1.16.9 h1:LQo3MUIOzod9JdUK+wxmSdgzLVYUbII3jXn3S/HJZU0= @@ -158,6 +160,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 h1:abKT+RuM1sdCNZIGIfZpLkvxEX3Rpsto019XG/rkYG8= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8/go.mod h1:Owc4ysUE71JSruVTTa3h4f2pp3E4hlcAtmeNXxDmjj4= github.com/aws/aws-sdk-go-v2/service/ec2 v1.138.2 h1:e3Imv1oXz+W3Tfclflkh72t5TUPUwWdkHP7ctQGk8Dc= github.com/aws/aws-sdk-go-v2/service/ec2 v1.138.2/go.mod h1:d1hAqgLDOPaSO1Piy/0bBmj6oAplFwv6p0cquHntNHM= github.com/aws/aws-sdk-go-v2/service/ec2instanceconnect v1.20.2 h1:owl9n1S8bJ4w/GpXvvwC9a/zpJnfvoOyvFWHKdSPJW8= @@ -168,8 +172,14 @@ github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.21.2 h1:BwF80uV9Ga4yB6UMUggQ4R github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.21.2/go.mod h1:IQcO27/ICcX5d2gBgh6sYHZ4hkcVn3n+0A6brZwOnWU= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 h1:xyfOAYV/ujzZOo01H9+OnyeiRKmTEp6EsITTsmq332Q= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8/go.mod h1:coLeQEoKzW9ViTL2bn0YUlU7K0RYjivKudG74gtd+sI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 h1:EamsKe+ZjkOQjDdHd86/JCEucjFKQ9T0atWKO4s2Lgs= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8/go.mod h1:Q0vV3/csTpbkfKLI5Sb56cJQTCTtJ0ixdb7P+Wedqiw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 h1:ip5ia3JOXl4OAsqeTdrOOmqKgoWiu+t9XSOnRzBwmRs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8/go.mod h1:kE+aERnK9VQIw1vrk7ElAvhCsgLNzGyCPNg2Qe4Eq4c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3 h1:j34+Cw6EzOZmk1V505oZimpNSco1e83K7HPQKxCc0wY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3/go.mod h1:thjZng67jGsvMyVZnSxlcqKyLwB0XTG8bHIRZPTJ+Bs= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.2 h1:JKbfiLwEqJp8zaOAOn6AVSMS96gdwP3TjBMvZYsbxqE= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.2/go.mod h1:pbBOMK8UicdDK11zsPSGbpFh9Xwbd1oD3t7pSxXgNxU= github.com/aws/aws-sdk-go-v2/service/ssm v1.44.2 h1:lmdmYCvG1EJKGLEsUsYDNO6MwZyBZROrRg04Vrb5TwA= diff --git a/policy/bundle.go b/policy/bundle.go index 56b0c3faf..5ca372a83 100644 --- a/policy/bundle.go +++ b/policy/bundle.go @@ -7,9 +7,6 @@ import ( "context" "fmt" "io" - "io/fs" - "os" - "path/filepath" "regexp" "sort" "strings" @@ -34,106 +31,57 @@ const ( MRN_RESOURCE_CONTROL = "controls" ) -// BundleExecutionChecksum creates a combined execution checksum from a policy -// and framework. Either may be nil. -func BundleExecutionChecksum(policy *Policy, framework *Framework) string { - res := checksums.New - if policy != nil { - res = res.Add(policy.GraphExecutionChecksum) - } - if framework != nil { - res = res.Add(framework.GraphExecutionChecksum) - } - return res.String() +type BundleResolver interface { + Load(ctx context.Context, path string) (*Bundle, error) + IsApplicable(path string) bool } -// BundleFromPaths loads a single policy bundle file or a bundle that -// was split into multiple files into a single PolicyBundle struct -func BundleFromPaths(paths ...string) (*Bundle, error) { - // load all the source files - resolvedFilenames, err := WalkPolicyBundleFiles(paths...) - if err != nil { - log.Error().Err(err).Msg("could not resolve bundle files") - return nil, err - } - - // aggregate all files into a single policy bundle - aggregatedBundle, err := aggregateFilesToBundle(resolvedFilenames) - if err != nil { - log.Debug().Err(err).Msg("could merge bundle files") - return nil, err - } - - logger.DebugDumpYAML("resolved_mql_bundle.mql", aggregatedBundle) - return aggregatedBundle, nil +// note: order matters here as we want to use the most specific resolver first +var bundleResolvers = []BundleResolver{ + &s3Resolver{}, + &fileResolver{}, } -// WalkPolicyBundleFiles iterates over all provided filenames and -// checks if the name is a file or a directory. If the filename -// is a directory, it walks the directory recursively -func WalkPolicyBundleFiles(filenames ...string) ([]string, error) { - // resolve file names - resolvedFilenames := []string{} - for i := range filenames { - filename := filenames[i] - fi, err := os.Stat(filename) - if err != nil { - return nil, errors.Wrap(err, "could not load policy bundle file: "+filename) - } - - if fi.IsDir() { - filepath.WalkDir(filename, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - // we ignore directories because WalkDir already walks them - if d.IsDir() { - return nil - } - - // only consider .yaml|.yml files - if strings.HasSuffix(d.Name(), ".mql.yaml") || strings.HasSuffix(d.Name(), ".mql.yml") { - resolvedFilenames = append(resolvedFilenames, path) - } - - return nil - }) - } else { - resolvedFilenames = append(resolvedFilenames, filename) +func getResolver(path string) BundleResolver { + for _, resolver := range bundleResolvers { + if resolver.IsApplicable(path) { + return resolver } } - - return resolvedFilenames, nil + // we fallback to using the file resolver if none of the resolvers are applicable + return &fileResolver{} } -// aggregateFilesToBundle iterates over all provided files and loads its content. -// It assumes that all provided files are checked upfront and are not a directory -func aggregateFilesToBundle(paths []string) (*Bundle, error) { - // iterate over all files, load them and merge them - mergedBundle := &Bundle{} - - for i := range paths { - path := paths[i] - log.Debug().Str("path", path).Msg("loading policy bundle file") - bundle, err := bundleFromSingleFile(path) +// iterates through all the resolvers until it finds an applicable one and then uses that to load the bundle +// from the provided path +func BundleFromPaths(paths ...string) (*Bundle, error) { + ctx := context.Background() + aggregatedBundle := &Bundle{} + for _, path := range paths { + resolver := getResolver(path) + bundle, err := resolver.Load(ctx, path) if err != nil { - return nil, errors.Wrap(err, "could not load file: "+path) + log.Error().Err(err).Msg("could not resolve bundle files") + return nil, err } - - mergedBundle = Merge(mergedBundle, bundle) + aggregatedBundle = Merge(aggregatedBundle, bundle) } - return mergedBundle, nil + logger.DebugDumpYAML("resolved_mql_bundle.mql", aggregatedBundle) + return aggregatedBundle, nil } -// bundleFromSingleFile loads a policy bundle from a single file -func bundleFromSingleFile(path string) (*Bundle, error) { - bundleData, err := os.ReadFile(path) - if err != nil { - return nil, err +// BundleExecutionChecksum creates a combined execution checksum from a policy +// and framework. Either may be nil. +func BundleExecutionChecksum(policy *Policy, framework *Framework) string { + res := checksums.New + if policy != nil { + res = res.Add(policy.GraphExecutionChecksum) } - - return BundleFromYAML(bundleData) + if framework != nil { + res = res.Add(framework.GraphExecutionChecksum) + } + return res.String() } // Merge combines two PolicyBundle and merges the data additive into one diff --git a/policy/bundle_file_resolver.go b/policy/bundle_file_resolver.go new file mode 100644 index 000000000..5b48739b8 --- /dev/null +++ b/policy/bundle_file_resolver.go @@ -0,0 +1,117 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package policy + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +type fileResolver struct{} + +func (l *fileResolver) Load(ctx context.Context, path string) (*Bundle, error) { + return loadBundlesFromPaths(path) +} + +func (r *fileResolver) IsApplicable(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// loadBundlesFromPaths loads a single policy bundle file or a bundle that +// was split into multiple files into a single PolicyBundle struct +func loadBundlesFromPaths(paths ...string) (*Bundle, error) { + // load all the source files + resolvedFilenames, err := WalkPolicyBundleFiles(paths...) + if err != nil { + log.Error().Err(err).Msg("could not resolve bundle files") + return nil, err + } + + // aggregate all files into a single policy bundle + aggregatedBundle, err := aggregateFilesToBundle(resolvedFilenames) + if err != nil { + log.Debug().Err(err).Msg("could not merge bundle files") + return nil, err + } + + return aggregatedBundle, nil +} + +// WalkPolicyBundleFiles iterates over all provided filenames and +// checks if the name is a file or a directory. If the filename +// is a directory, it walks the directory recursively +func WalkPolicyBundleFiles(filenames ...string) ([]string, error) { + // resolve file names + resolvedFilenames := []string{} + for i := range filenames { + filename := filenames[i] + fi, err := os.Stat(filename) + if err != nil { + return nil, errors.Wrap(err, "could not load policy bundle file: "+filename) + } + + if fi.IsDir() { + err := filepath.WalkDir(filename, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + // we ignore directories because WalkDir already walks them + if d.IsDir() { + return nil + } + + // only consider .yaml|.yml files + if strings.HasSuffix(d.Name(), ".mql.yaml") || strings.HasSuffix(d.Name(), ".mql.yml") { + resolvedFilenames = append(resolvedFilenames, path) + } + + return nil + }) + if err != nil { + return nil, err + } + } else { + resolvedFilenames = append(resolvedFilenames, filename) + } + } + + return resolvedFilenames, nil +} + +// aggregateFilesToBundle iterates over all provided files and loads its content. +// It assumes that all provided files are checked upfront and are not a directory +func aggregateFilesToBundle(paths []string) (*Bundle, error) { + // iterate over all files, load them and merge them + mergedBundle := &Bundle{} + + for i := range paths { + path := paths[i] + log.Debug().Str("path", path).Msg("local>loading policy bundle file") + bundle, err := bundleFromSingleFile(path) + if err != nil { + return nil, errors.Wrap(err, "could not load file: "+path) + } + + mergedBundle = Merge(mergedBundle, bundle) + } + + return mergedBundle, nil +} + +// bundleFromSingleFile loads a policy bundle from a single file +func bundleFromSingleFile(path string) (*Bundle, error) { + bundleData, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + return BundleFromYAML(bundleData) +} diff --git a/policy/bundle_s3_resolver.go b/policy/bundle_s3_resolver.go new file mode 100644 index 000000000..0354dbe13 --- /dev/null +++ b/policy/bundle_s3_resolver.go @@ -0,0 +1,78 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package policy + +import ( + "context" + "io" + "strings" + + awsconfigv2 "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/rs/zerolog/log" +) + +type s3Resolver struct{} + +func (l *s3Resolver) Load(ctx context.Context, path string) (*Bundle, error) { + cfg, err := awsconfigv2.LoadDefaultConfig(ctx) + if err != nil { + return nil, err + } + s3Client := s3.NewFromConfig(cfg) + + trimmed := strings.TrimPrefix(path, "s3://") + if ok, bucket, key := isS3Key(trimmed); ok { + return fetchBundleFromS3(ctx, s3Client, bucket, key) + } + + // else we're looking at a bucket so we need to iterate over all the files inside + files, err := s3Client.ListObjects(ctx, &s3.ListObjectsInput{Bucket: &trimmed}) + if err != nil { + return nil, err + } + mergedBundle := &Bundle{} + resolvedNames := []string{} + for _, file := range files.Contents { + key := *file.Key + if strings.HasSuffix(key, ".mql.yaml") || strings.HasSuffix(key, ".mql.yml") { + resolvedNames = append(resolvedNames, key) + } + } + for _, file := range resolvedNames { + bundle, err := fetchBundleFromS3(ctx, s3Client, trimmed, file) + if err != nil { + return nil, err + } + mergedBundle = Merge(mergedBundle, bundle) + } + return mergedBundle, nil +} + +func (r *s3Resolver) IsApplicable(path string) bool { + return strings.HasPrefix(path, "s3://") +} + +// basic check that validates if the provided path is a s3 key. it returns the bucket and the key that can be used to fetch the object +// an s3 key is defined as a path that contains more than one slash, e.g. my-bucket/my-key +func isS3Key(path string) (bool, string, string) { + parts := strings.Split(path, "/") + if len(parts) > 1 { + return true, parts[0], strings.Join(parts[1:], "/") + } + return false, "", "" +} + +func fetchBundleFromS3(ctx context.Context, client *s3.Client, bucket string, key string) (*Bundle, error) { + resp, err := client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &key}) + if err != nil { + return nil, err + } + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + log.Debug().Str("bucket", bucket).Str("key", key).Msg("s3>loaded bundle file from s3") + return BundleFromYAML(data) +}