From a7e8a329c60b9ef63d00104d09fc3a1708d3222d Mon Sep 17 00:00:00 2001 From: Preslav Gerchev Date: Fri, 8 Dec 2023 12:42:04 +0200 Subject: [PATCH] =?UTF-8?q?=E2=AD=90=EF=B8=8F=20Allow=20loading=20policies?= =?UTF-8?q?=20from=20an=20s3=20bucket.=20(#988)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ⭐️ Allow loading policies from an s3 bucket. Signed-off-by: Preslav * Add s3 tests with an s3 fake. Update example policies to v8 format. --------- Signed-off-by: Preslav --- apps/cnspec/cmd/bundle.go | 6 +- apps/cnspec/cmd/scan.go | 3 +- examples/directory/example1.mql.yaml | 17 ++- examples/directory/example2.mql.yaml | 11 +- go.mod | 7 +- go.sum | 10 ++ internal/bundle/lint.go | 3 +- policy/bundle.go | 132 +++++++------------ policy/bundle_file_resolver.go | 125 ++++++++++++++++++ policy/bundle_s3_resolver.go | 96 ++++++++++++++ policy/bundle_test.go | 185 +++++++++++++++++++++++++-- policy/cnspec_policy.pb.go | 2 +- policy/policy_test.go | 3 +- policy/scan/local_scanner_test.go | 15 ++- 14 files changed, 494 insertions(+), 121 deletions(-) create mode 100644 policy/bundle_file_resolver.go create mode 100644 policy/bundle_s3_resolver.go diff --git a/apps/cnspec/cmd/bundle.go b/apps/cnspec/cmd/bundle.go index 07e03f6b..f80736c8 100644 --- a/apps/cnspec/cmd/bundle.go +++ b/apps/cnspec/cmd/bundle.go @@ -204,7 +204,8 @@ var policyPublishCmd = &cobra.Command{ } // compile manipulates the bundle, therefore we read it again - policyBundle, err := policy.BundleFromPaths(filename) + bundleLoader := policy.DefaultBundleLoader() + policyBundle, err := bundleLoader.BundleFromPaths(filename) if err != nil { log.Fatal().Err(err).Msg("could not load policy bundle") } @@ -271,7 +272,8 @@ var policyDocsCmd = &cobra.Command{ viper.BindPFlag("no-code", cmd.Flags().Lookup("no-code")) }, Run: func(cmd *cobra.Command, args []string) { - bundle, err := policy.BundleFromPaths(args...) + bundleLoader := policy.DefaultBundleLoader() + bundle, err := bundleLoader.BundleFromPaths(args...) if err != nil { log.Fatal().Err(err).Msg("failed to load bundle") } diff --git a/apps/cnspec/cmd/scan.go b/apps/cnspec/cmd/scan.go index c8a26458..b68ffec5 100644 --- a/apps/cnspec/cmd/scan.go +++ b/apps/cnspec/cmd/scan.go @@ -319,7 +319,8 @@ func (c *scanConfig) loadPolicies() error { return nil } - bundle, err := policy.BundleFromPaths(c.PolicyPaths...) + bundleLoader := policy.DefaultBundleLoader() + bundle, err := bundleLoader.BundleFromPaths(c.PolicyPaths...) if err != nil { return err } diff --git a/examples/directory/example1.mql.yaml b/examples/directory/example1.mql.yaml index 23de7ed4..8701302c 100644 --- a/examples/directory/example1.mql.yaml +++ b/examples/directory/example1.mql.yaml @@ -8,12 +8,11 @@ policies: authors: - name: Mondoo email: hello@mondoo.com - specs: - - asset_filter: - query: platform.family.contains(_ == 'unix') - scoring_queries: - sshd-01: {} - sshd-02: {} - sshd-03: {} - data_queries: - sshd-d-1: 0 + groups: + - filters: platform.family.contains('unix') + checks: + - uid: sshd-01 + - uid: sshd-02 + - uid: sshd-03 + queries: + - uid: sshd-d-1 diff --git a/examples/directory/example2.mql.yaml b/examples/directory/example2.mql.yaml index 64626875..aef3acf4 100644 --- a/examples/directory/example2.mql.yaml +++ b/examples/directory/example2.mql.yaml @@ -5,10 +5,9 @@ policies: - uid: example2 name: Another policy version: 1.0.0 - specs: - - asset_filter: - query: platform.family.contains(_ == 'unix') + groups: + - filters: platform.family.contains('unix') + checks: + - uid: linux-1 policies: - example1: {} - scoring_queries: - linux-1: {} + - uid: example1 \ No newline at end of file diff --git a/go.mod b/go.mod index aceca6a3..ab5c7860 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require go.mondoo.com/cnquery/v9 v9.10.0 require ( github.com/Masterminds/semver v1.5.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3 github.com/cockroachdb/errors v1.11.1 github.com/google/go-cmdtest v0.4.0 github.com/google/uuid v1.4.0 @@ -56,9 +57,13 @@ require ( github.com/alecthomas/go-check-sumtype v0.1.3 // indirect github.com/alexkohler/nakedret/v2 v2.0.2 // indirect github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 // indirect github.com/aws/aws-sdk-go-v2/service/ecr v1.24.2 // indirect github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.21.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 // indirect github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20231121224113-b6714ac5eb13 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect @@ -144,7 +149,7 @@ require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go v1.48.12 // indirect github.com/aws/aws-sdk-go-v2 v1.23.5 // indirect - github.com/aws/aws-sdk-go-v2/config v1.25.11 // indirect + github.com/aws/aws-sdk-go-v2/config v1.25.11 github.com/aws/aws-sdk-go-v2/credentials v1.16.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect diff --git a/go.sum b/go.sum index 88997e66..0a1f11c2 100644 --- a/go.sum +++ b/go.sum @@ -146,6 +146,8 @@ github.com/aws/aws-sdk-go v1.48.12 h1:n+eGzflzzvYubu2cOjqpVll7lF+Ci0ThyCpg5kzfzb github.com/aws/aws-sdk-go v1.48.12/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.23.5 h1:xK6C4udTyDMd82RFvNkDQxtAd00xlzFUtX4fF2nMZyg= github.com/aws/aws-sdk-go-v2 v1.23.5/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3 h1:Zx9+31KyB8wQna6SXFWOewlgoY5uGdDAu6PTOEU3OQI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.3/go.mod h1:zxbEJhRdKTH1nqS2qu6UJ7zGe25xaHxZXaC2CvuQFnA= github.com/aws/aws-sdk-go-v2/config v1.25.11 h1:RWzp7jhPRliIcACefGkKp03L0Yofmd2p8M25kbiyvno= github.com/aws/aws-sdk-go-v2/config v1.25.11/go.mod h1:BVUs0chMdygHsQtvaMyEOpW2GIW+ubrxJLgIz/JU29s= github.com/aws/aws-sdk-go-v2/credentials v1.16.9 h1:LQo3MUIOzod9JdUK+wxmSdgzLVYUbII3jXn3S/HJZU0= @@ -158,6 +160,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8 h1:abKT+RuM1sdCNZIGIfZpLkvxEX3Rpsto019XG/rkYG8= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.8/go.mod h1:Owc4ysUE71JSruVTTa3h4f2pp3E4hlcAtmeNXxDmjj4= github.com/aws/aws-sdk-go-v2/service/ec2 v1.138.2 h1:e3Imv1oXz+W3Tfclflkh72t5TUPUwWdkHP7ctQGk8Dc= github.com/aws/aws-sdk-go-v2/service/ec2 v1.138.2/go.mod h1:d1hAqgLDOPaSO1Piy/0bBmj6oAplFwv6p0cquHntNHM= github.com/aws/aws-sdk-go-v2/service/ec2instanceconnect v1.20.2 h1:owl9n1S8bJ4w/GpXvvwC9a/zpJnfvoOyvFWHKdSPJW8= @@ -168,8 +172,14 @@ github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.21.2 h1:BwF80uV9Ga4yB6UMUggQ4R github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.21.2/go.mod h1:IQcO27/ICcX5d2gBgh6sYHZ4hkcVn3n+0A6brZwOnWU= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8 h1:xyfOAYV/ujzZOo01H9+OnyeiRKmTEp6EsITTsmq332Q= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.8/go.mod h1:coLeQEoKzW9ViTL2bn0YUlU7K0RYjivKudG74gtd+sI= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 h1:EamsKe+ZjkOQjDdHd86/JCEucjFKQ9T0atWKO4s2Lgs= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8/go.mod h1:Q0vV3/csTpbkfKLI5Sb56cJQTCTtJ0ixdb7P+Wedqiw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8 h1:ip5ia3JOXl4OAsqeTdrOOmqKgoWiu+t9XSOnRzBwmRs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.8/go.mod h1:kE+aERnK9VQIw1vrk7ElAvhCsgLNzGyCPNg2Qe4Eq4c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3 h1:j34+Cw6EzOZmk1V505oZimpNSco1e83K7HPQKxCc0wY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.47.3/go.mod h1:thjZng67jGsvMyVZnSxlcqKyLwB0XTG8bHIRZPTJ+Bs= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.2 h1:JKbfiLwEqJp8zaOAOn6AVSMS96gdwP3TjBMvZYsbxqE= github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.25.2/go.mod h1:pbBOMK8UicdDK11zsPSGbpFh9Xwbd1oD3t7pSxXgNxU= github.com/aws/aws-sdk-go-v2/service/ssm v1.44.2 h1:lmdmYCvG1EJKGLEsUsYDNO6MwZyBZROrRg04Vrb5TwA= diff --git a/internal/bundle/lint.go b/internal/bundle/lint.go index fb88c639..360a7f2c 100644 --- a/internal/bundle/lint.go +++ b/internal/bundle/lint.go @@ -171,7 +171,8 @@ func Lint(schema llx.Schema, files ...string) (*Results, error) { // Note: we only run compile on the aggregated level to ensure the bundle in combination is valid // Invalid yaml files are already caught by the individual linting, therefore we do not need extra error handling here - policyBundle, err := policy.BundleFromPaths(files...) + bundleLoader := policy.DefaultBundleLoader() + policyBundle, err := bundleLoader.BundleFromPaths(files...) if err == nil { _, err = policyBundle.Compile(context.Background(), schema, nil) if err != nil { diff --git a/policy/bundle.go b/policy/bundle.go index 56b0c3fa..b5476feb 100644 --- a/policy/bundle.go +++ b/policy/bundle.go @@ -7,9 +7,6 @@ import ( "context" "fmt" "io" - "io/fs" - "os" - "path/filepath" "regexp" "sort" "strings" @@ -34,106 +31,71 @@ const ( MRN_RESOURCE_CONTROL = "controls" ) -// BundleExecutionChecksum creates a combined execution checksum from a policy -// and framework. Either may be nil. -func BundleExecutionChecksum(policy *Policy, framework *Framework) string { - res := checksums.New - if policy != nil { - res = res.Add(policy.GraphExecutionChecksum) - } - if framework != nil { - res = res.Add(framework.GraphExecutionChecksum) - } - return res.String() +type BundleResolver interface { + Load(ctx context.Context, path string) (*Bundle, error) + IsApplicable(path string) bool } -// BundleFromPaths loads a single policy bundle file or a bundle that -// was split into multiple files into a single PolicyBundle struct -func BundleFromPaths(paths ...string) (*Bundle, error) { - // load all the source files - resolvedFilenames, err := WalkPolicyBundleFiles(paths...) - if err != nil { - log.Error().Err(err).Msg("could not resolve bundle files") - return nil, err - } - - // aggregate all files into a single policy bundle - aggregatedBundle, err := aggregateFilesToBundle(resolvedFilenames) - if err != nil { - log.Debug().Err(err).Msg("could merge bundle files") - return nil, err - } - - logger.DebugDumpYAML("resolved_mql_bundle.mql", aggregatedBundle) - return aggregatedBundle, nil +type BundleLoader struct { + resolvers []BundleResolver } -// WalkPolicyBundleFiles iterates over all provided filenames and -// checks if the name is a file or a directory. If the filename -// is a directory, it walks the directory recursively -func WalkPolicyBundleFiles(filenames ...string) ([]string, error) { - // resolve file names - resolvedFilenames := []string{} - for i := range filenames { - filename := filenames[i] - fi, err := os.Stat(filename) - if err != nil { - return nil, errors.Wrap(err, "could not load policy bundle file: "+filename) - } - - if fi.IsDir() { - filepath.WalkDir(filename, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - // we ignore directories because WalkDir already walks them - if d.IsDir() { - return nil - } +func NewBundleLoader(resolvers ...BundleResolver) *BundleLoader { + return &BundleLoader{resolvers: resolvers} +} - // only consider .yaml|.yml files - if strings.HasSuffix(d.Name(), ".mql.yaml") || strings.HasSuffix(d.Name(), ".mql.yml") { - resolvedFilenames = append(resolvedFilenames, path) - } +func DefaultBundleLoader() *BundleLoader { + return NewBundleLoader(defaultS3BundleResolver(), defaultFileBundleResolver()) +} - return nil - }) - } else { - resolvedFilenames = append(resolvedFilenames, filename) +func (l *BundleLoader) getResolver(path string) (BundleResolver, error) { + for _, resolver := range l.resolvers { + if resolver.IsApplicable(path) { + return resolver, nil } } - - return resolvedFilenames, nil + return nil, fmt.Errorf("no resolver found for path '%s'", path) } -// aggregateFilesToBundle iterates over all provided files and loads its content. -// It assumes that all provided files are checked upfront and are not a directory -func aggregateFilesToBundle(paths []string) (*Bundle, error) { - // iterate over all files, load them and merge them - mergedBundle := &Bundle{} +// Deprecated: Use BundleLoader.BundleFromPaths instead +func BundleFromPaths(paths ...string) (*Bundle, error) { + defaultLoader := DefaultBundleLoader() + return defaultLoader.BundleFromPaths(paths...) +} - for i := range paths { - path := paths[i] - log.Debug().Str("path", path).Msg("loading policy bundle file") - bundle, err := bundleFromSingleFile(path) +// iterates through all the resolvers until it finds an applicable one and then uses that to load the bundle +// from the provided path +func (l *BundleLoader) BundleFromPaths(paths ...string) (*Bundle, error) { + ctx := context.Background() + aggregatedBundle := &Bundle{} + for _, path := range paths { + resolver, err := l.getResolver(path) if err != nil { - return nil, errors.Wrap(err, "could not load file: "+path) + return nil, err } - - mergedBundle = Merge(mergedBundle, bundle) + bundle, err := resolver.Load(ctx, path) + if err != nil { + log.Error().Err(err).Msg("could not resolve bundle files") + return nil, err + } + aggregatedBundle = Merge(aggregatedBundle, bundle) } - return mergedBundle, nil + logger.DebugDumpYAML("resolved_mql_bundle.mql", aggregatedBundle) + return aggregatedBundle, nil } -// bundleFromSingleFile loads a policy bundle from a single file -func bundleFromSingleFile(path string) (*Bundle, error) { - bundleData, err := os.ReadFile(path) - if err != nil { - return nil, err +// BundleExecutionChecksum creates a combined execution checksum from a policy +// and framework. Either may be nil. +func BundleExecutionChecksum(policy *Policy, framework *Framework) string { + res := checksums.New + if policy != nil { + res = res.Add(policy.GraphExecutionChecksum) } - - return BundleFromYAML(bundleData) + if framework != nil { + res = res.Add(framework.GraphExecutionChecksum) + } + return res.String() } // Merge combines two PolicyBundle and merges the data additive into one diff --git a/policy/bundle_file_resolver.go b/policy/bundle_file_resolver.go new file mode 100644 index 00000000..2f96f449 --- /dev/null +++ b/policy/bundle_file_resolver.go @@ -0,0 +1,125 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package policy + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "github.com/rs/zerolog/log" +) + +type fileBundleResolver struct{} + +func defaultFileBundleResolver() *fileBundleResolver { + return NewFileBundleResolver() +} + +func (l *fileBundleResolver) Load(ctx context.Context, path string) (*Bundle, error) { + return loadBundlesFromPaths(path) +} + +func NewFileBundleResolver() *fileBundleResolver { + return &fileBundleResolver{} +} + +func (r *fileBundleResolver) IsApplicable(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// loadBundlesFromPaths loads a single policy bundle file or a bundle that +// was split into multiple files into a single PolicyBundle struct +func loadBundlesFromPaths(paths ...string) (*Bundle, error) { + // load all the source files + resolvedFilenames, err := WalkPolicyBundleFiles(paths...) + if err != nil { + log.Error().Err(err).Msg("could not resolve bundle files") + return nil, err + } + + // aggregate all files into a single policy bundle + aggregatedBundle, err := aggregateFilesToBundle(resolvedFilenames) + if err != nil { + log.Debug().Err(err).Msg("could not merge bundle files") + return nil, err + } + + return aggregatedBundle, nil +} + +// WalkPolicyBundleFiles iterates over all provided filenames and +// checks if the name is a file or a directory. If the filename +// is a directory, it walks the directory recursively +func WalkPolicyBundleFiles(filenames ...string) ([]string, error) { + // resolve file names + resolvedFilenames := []string{} + for i := range filenames { + filename := filenames[i] + fi, err := os.Stat(filename) + if err != nil { + return nil, errors.Wrap(err, "could not load policy bundle file: "+filename) + } + + if fi.IsDir() { + err := filepath.WalkDir(filename, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + // we ignore directories because WalkDir already walks them + if d.IsDir() { + return nil + } + + // only consider .yaml|.yml files + if strings.HasSuffix(d.Name(), ".mql.yaml") || strings.HasSuffix(d.Name(), ".mql.yml") { + resolvedFilenames = append(resolvedFilenames, path) + } + + return nil + }) + if err != nil { + return nil, err + } + } else { + resolvedFilenames = append(resolvedFilenames, filename) + } + } + + return resolvedFilenames, nil +} + +// aggregateFilesToBundle iterates over all provided files and loads its content. +// It assumes that all provided files are checked upfront and are not a directory +func aggregateFilesToBundle(paths []string) (*Bundle, error) { + // iterate over all files, load them and merge them + mergedBundle := &Bundle{} + + for i := range paths { + path := paths[i] + log.Debug().Str("path", path).Msg("local>loading policy bundle file") + bundle, err := bundleFromSingleFile(path) + if err != nil { + return nil, errors.Wrap(err, "could not load file: "+path) + } + + mergedBundle = Merge(mergedBundle, bundle) + } + + return mergedBundle, nil +} + +// bundleFromSingleFile loads a policy bundle from a single file +func bundleFromSingleFile(path string) (*Bundle, error) { + bundleData, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + return BundleFromYAML(bundleData) +} diff --git a/policy/bundle_s3_resolver.go b/policy/bundle_s3_resolver.go new file mode 100644 index 00000000..09d5056a --- /dev/null +++ b/policy/bundle_s3_resolver.go @@ -0,0 +1,96 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package policy + +import ( + "context" + "io" + "strings" + + awsconfigv2 "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/rs/zerolog/log" +) + +type S3GetObjectAPI interface { + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + ListObjects(ctx context.Context, params *s3.ListObjectsInput, optFns ...func(*s3.Options)) (*s3.ListObjectsOutput, error) +} + +type s3BundleResolver struct { + s3Client S3GetObjectAPI +} + +func defaultS3BundleResolver() *s3BundleResolver { + return &s3BundleResolver{} +} + +func NewS3BundleResolver(api S3GetObjectAPI) *s3BundleResolver { + return &s3BundleResolver{s3Client: api} +} + +func (r *s3BundleResolver) Load(ctx context.Context, path string) (*Bundle, error) { + // if we have no explicitly provided s3 client, we need to create one. we can optimize this later on by ensuring we require a client + // when creating the s3 bundle resolver + if r.s3Client == nil { + cfg, err := awsconfigv2.LoadDefaultConfig(ctx) + if err != nil { + return nil, err + } + r.s3Client = s3.NewFromConfig(cfg) + } + trimmed := strings.TrimPrefix(path, "s3://") + if ok, bucket, key := isS3Key(trimmed); ok { + return r.fetchBundleFromS3(ctx, bucket, key) + } + + // else we're looking at a bucket so we need to iterate over all the files inside + files, err := r.s3Client.ListObjects(ctx, &s3.ListObjectsInput{Bucket: &trimmed}) + if err != nil { + return nil, err + } + mergedBundle := &Bundle{} + resolvedNames := []string{} + for _, file := range files.Contents { + key := *file.Key + if strings.HasSuffix(key, ".mql.yaml") || strings.HasSuffix(key, ".mql.yml") { + resolvedNames = append(resolvedNames, key) + } + } + for _, file := range resolvedNames { + bundle, err := r.fetchBundleFromS3(ctx, trimmed, file) + if err != nil { + return nil, err + } + mergedBundle = Merge(mergedBundle, bundle) + } + return mergedBundle, nil +} + +func (r *s3BundleResolver) IsApplicable(path string) bool { + return strings.HasPrefix(path, "s3://") +} + +// basic check that validates if the provided path is a s3 key. it returns the bucket and the key that can be used to fetch the object +// an s3 key is defined as a path that contains more than one slash, e.g. my-bucket/my-key +func isS3Key(path string) (bool, string, string) { + parts := strings.Split(path, "/") + if len(parts) > 1 { + return true, parts[0], strings.Join(parts[1:], "/") + } + return false, "", "" +} + +func (r *s3BundleResolver) fetchBundleFromS3(ctx context.Context, bucket string, key string) (*Bundle, error) { + resp, err := r.s3Client.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &key}) + if err != nil { + return nil, err + } + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + log.Debug().Str("bucket", bucket).Str("key", key).Msg("s3>loaded bundle file from s3") + return BundleFromYAML(data) +} diff --git a/policy/bundle_test.go b/policy/bundle_test.go index 84ee81e1..ce850404 100644 --- a/policy/bundle_test.go +++ b/policy/bundle_test.go @@ -5,9 +5,14 @@ package policy_test import ( "context" + "errors" + "io" + "os" "strings" "testing" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.mondoo.com/cnquery/v9/explorer" @@ -17,9 +22,46 @@ import ( "go.mondoo.com/cnspec/v9/policy" ) -func TestBundleFromPaths(t *testing.T) { +type s3Fake struct { + bucketObjects map[string]map[string][]byte +} + +func (s *s3Fake) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + key := *params.Key + bucket := *params.Bucket + items := s.bucketObjects[bucket] + if items == nil { + return nil, errors.New("not found") + } + if data, ok := items[key]; ok { + return &s3.GetObjectOutput{ + Body: io.NopCloser(strings.NewReader(string(data))), + }, nil + } + return nil, errors.New("not found") +} + +func (s *s3Fake) ListObjects(ctx context.Context, params *s3.ListObjectsInput, optFns ...func(*s3.Options)) (*s3.ListObjectsOutput, error) { + var objects []types.Object + bucket := *params.Bucket + items := s.bucketObjects[bucket] + if items == nil { + return nil, errors.New("not found") + } + for k := range items { + objects = append(objects, types.Object{ + Key: &k, + }) + } + return &s3.ListObjectsOutput{ + Contents: objects, + }, nil +} + +func TestBundleFromLocal(t *testing.T) { t.Run("mql bundle file with multiple queries", func(t *testing.T) { - bundle, err := policy.BundleFromPaths("../examples/example.mql.yaml") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("../examples/example.mql.yaml") require.NoError(t, err) require.NotNil(t, bundle) assert.Len(t, bundle.Queries, 1) @@ -30,7 +72,8 @@ func TestBundleFromPaths(t *testing.T) { }) t.Run("mql bundle file with multiple policies and queries", func(t *testing.T) { - bundle, err := policy.BundleFromPaths("../examples/complex.mql.yaml") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("../examples/complex.mql.yaml") require.NoError(t, err) require.NotNil(t, bundle) assert.Len(t, bundle.Queries, 5) @@ -38,19 +81,141 @@ func TestBundleFromPaths(t *testing.T) { }) t.Run("mql bundle file with directory structure", func(t *testing.T) { - bundle, err := policy.BundleFromPaths("../examples/directory") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("../examples/directory") + require.NoError(t, err) + require.NotNil(t, bundle) + assert.Len(t, bundle.Queries, 5) + assert.Len(t, bundle.Policies, 2) + }) +} + +func TestBundleFromS3(t *testing.T) { + t.Run("mql bundle file with multiple queries via a specific s3 key", func(t *testing.T) { + s3Fake := &s3Fake{ + bucketObjects: map[string]map[string][]byte{}, + } + data, err := os.ReadFile("../examples/example.mql.yaml") + require.NoError(t, err) + s3Fake.bucketObjects["test-bucket"] = map[string][]byte{"example.mql.yaml": data} + + loader := policy.NewBundleLoader(policy.NewS3BundleResolver(s3Fake)) + bundle, err := loader.BundleFromPaths("s3://test-bucket/example.mql.yaml") + + require.NoError(t, err) + require.NotNil(t, bundle) + assert.Len(t, bundle.Queries, 1) + require.Len(t, bundle.Policies, 1) + require.Len(t, bundle.Policies[0].Groups, 1) + assert.Len(t, bundle.Policies[0].Groups[0].Checks, 3) + assert.Len(t, bundle.Policies[0].Groups[0].Queries, 2) + }) + + t.Run("mql bundle file with multiple policies and queries via a specific s3 key", func(t *testing.T) { + s3Fake := &s3Fake{ + bucketObjects: map[string]map[string][]byte{}, + } + data, err := os.ReadFile("../examples/complex.mql.yaml") + require.NoError(t, err) + + s3Fake.bucketObjects["test-bucket"] = map[string][]byte{"complex.mql.yaml": data} + + loader := policy.NewBundleLoader(policy.NewS3BundleResolver(s3Fake)) + bundle, err := loader.BundleFromPaths("s3://test-bucket/complex.mql.yaml") require.NoError(t, err) require.NotNil(t, bundle) assert.Len(t, bundle.Queries, 5) assert.Len(t, bundle.Policies, 2) }) + + t.Run("mql bundle file via an entire s3 bucket", func(t *testing.T) { + s3Fake := &s3Fake{ + bucketObjects: map[string]map[string][]byte{}, + } + data1, err := os.ReadFile("../examples/directory/example1.mql.yaml") + require.NoError(t, err) + data2, err := os.ReadFile("../examples/directory/example2.mql.yaml") + require.NoError(t, err) + s3Fake.bucketObjects["test-bucket"] = map[string][]byte{"example1.mql.yaml": data1, "example2.mql.yaml": data2} + + loader := policy.NewBundleLoader(policy.NewS3BundleResolver(s3Fake)) + bundle, err := loader.BundleFromPaths("s3://test-bucket") + require.NoError(t, err) + require.NotNil(t, bundle) + assert.Len(t, bundle.Policies, 2) + }) +} + +func TestBundleFromMixedSources(t *testing.T) { + t.Run("mql bundle file via a local file and a s3 key", func(t *testing.T) { + s3Fake := &s3Fake{ + bucketObjects: map[string]map[string][]byte{}, + } + data, err := os.ReadFile("../examples/directory/example1.mql.yaml") + require.NoError(t, err) + s3Fake.bucketObjects["test-bucket"] = map[string][]byte{"example1.mql.yaml": data} + + loader := policy.NewBundleLoader(policy.NewS3BundleResolver(s3Fake), policy.NewFileBundleResolver()) + bundle, err := loader.BundleFromPaths("s3://test-bucket/example1.mql.yaml", "../examples/directory/example2.mql.yaml") + require.NoError(t, err) + require.NotNil(t, bundle) + assert.Len(t, bundle.Policies, 2) + }) + + t.Run("mql bundle file via a local file and a s3 bucket", func(t *testing.T) { + s3Fake := &s3Fake{ + bucketObjects: map[string]map[string][]byte{}, + } + data, err := os.ReadFile("../examples/directory/example1.mql.yaml") + require.NoError(t, err) + s3Fake.bucketObjects["test-bucket"] = map[string][]byte{"example1.mql.yaml": data} + + loader := policy.NewBundleLoader(policy.NewS3BundleResolver(s3Fake), policy.NewFileBundleResolver()) + bundle, err := loader.BundleFromPaths("s3://test-bucket", "../examples/directory/example2.mql.yaml") + require.NoError(t, err) + require.NotNil(t, bundle) + assert.Len(t, bundle.Policies, 2) + }) + + t.Run("mql bundle file via a directory and a s3 key", func(t *testing.T) { + s3Fake := &s3Fake{ + bucketObjects: map[string]map[string][]byte{}, + } + data, err := os.ReadFile("../examples/directory/example1.mql.yaml") + require.NoError(t, err) + s3Fake.bucketObjects["test-bucket"] = map[string][]byte{"example1.mql.yaml": data} + + loader := policy.NewBundleLoader(policy.NewS3BundleResolver(s3Fake), policy.NewFileBundleResolver()) + bundle, err := loader.BundleFromPaths("s3://test-bucket/example1.mql.yaml", "../examples/directory/queries") + require.NoError(t, err) + require.NotNil(t, bundle) + assert.Len(t, bundle.Queries, 5) + assert.Len(t, bundle.Policies, 1) + }) + + t.Run("mql bundle file via a directory and a s3 bucket", func(t *testing.T) { + s3Fake := &s3Fake{ + bucketObjects: map[string]map[string][]byte{}, + } + data, err := os.ReadFile("../examples/directory/example1.mql.yaml") + require.NoError(t, err) + s3Fake.bucketObjects["test-bucket"] = map[string][]byte{"example1.mql.yaml": data} + + loader := policy.NewBundleLoader(policy.NewS3BundleResolver(s3Fake), policy.NewFileBundleResolver()) + bundle, err := loader.BundleFromPaths("s3://test-bucket", "../examples/directory/queries") + require.NoError(t, err) + require.NotNil(t, bundle) + assert.Len(t, bundle.Queries, 5) + assert.Len(t, bundle.Policies, 1) + }) } func TestPolicyBundleSort(t *testing.T) { - pb, err := policy.BundleFromPaths("./testdata/policybundle-deps.mql.yaml") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("./testdata/policybundle-deps.mql.yaml") require.NoError(t, err) - assert.Equal(t, 3, len(pb.Policies)) - pbm := pb.ToMap() + assert.Equal(t, 3, len(bundle.Policies)) + pbm := bundle.ToMap() policies, err := pbm.PoliciesSortedByDependency() require.NoError(t, err) @@ -62,7 +227,8 @@ func TestPolicyBundleSort(t *testing.T) { } func TestBundleCompile(t *testing.T) { - bundle, err := policy.BundleFromPaths("../examples/complex.mql.yaml") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("../examples/complex.mql.yaml") require.NoError(t, err) require.NotNil(t, bundle) @@ -186,7 +352,8 @@ func TestBundleCompile_FromQueryPackBundle(t *testing.T) { } func TestStableMqueryChecksum(t *testing.T) { - bundle, err := policy.BundleFromPaths("../examples/complex.mql.yaml") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("../examples/complex.mql.yaml") require.NoError(t, err) require.NotNil(t, bundle) diff --git a/policy/cnspec_policy.pb.go b/policy/cnspec_policy.pb.go index 352ed693..8c00c2ea 100644 --- a/policy/cnspec_policy.pb.go +++ b/policy/cnspec_policy.pb.go @@ -4,7 +4,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.25.0 +// protoc v4.25.1 // source: cnspec_policy.proto package policy diff --git a/policy/policy_test.go b/policy/policy_test.go index ad278054..fac52d08 100644 --- a/policy/policy_test.go +++ b/policy/policy_test.go @@ -87,7 +87,8 @@ func TestPolicyChecksums(t *testing.T) { for _, file := range files { t.Run(file, func(t *testing.T) { - b, err := policy.BundleFromPaths(file) + loader := policy.DefaultBundleLoader() + b, err := loader.BundleFromPaths(file) require.NoError(t, err) // check that the checksum is identical diff --git a/policy/scan/local_scanner_test.go b/policy/scan/local_scanner_test.go index 0544cadf..68990681 100644 --- a/policy/scan/local_scanner_test.go +++ b/policy/scan/local_scanner_test.go @@ -150,7 +150,8 @@ func (s *LocalScannerSuite) BeforeTest(suiteName, testName string) { } func (s *LocalScannerSuite) TestRunIncognito_SharedQuery() { - bundle, err := policy.BundleFromPaths("./testdata/shared-query.mql.yaml") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("./testdata/shared-query.mql.yaml") s.Require().NoError(err) _, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{ @@ -193,7 +194,8 @@ func (s *LocalScannerSuite) TestRunIncognito_SharedQuery() { } func (s *LocalScannerSuite) TestRunIncognito_ExceptionGroups() { - bundle, err := policy.BundleFromPaths("./testdata/exception-groups.mql.yaml") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("./testdata/exception-groups.mql.yaml") s.Require().NoError(err) _, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{ @@ -254,7 +256,8 @@ func (s *LocalScannerSuite) TestRunIncognito_ExceptionGroups() { } func (s *LocalScannerSuite) TestRunIncognito_ExceptionGroups_RejectedReview() { - bundle, err := policy.BundleFromPaths("./testdata/exception-groups.mql.yaml") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("./testdata/exception-groups.mql.yaml") s.Require().NoError(err) bundle.Policies[1].Groups[0].ReviewStatus = policy.ReviewStatus_REJECTED @@ -319,7 +322,8 @@ func (s *LocalScannerSuite) TestRunIncognito_ExceptionGroups_RejectedReview() { } func (s *LocalScannerSuite) TestRunIncognito_QueryExceptions() { - bundle, err := policy.BundleFromPaths("./testdata/exceptions.mql.yaml") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("./testdata/exceptions.mql.yaml") s.Require().NoError(err) _, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{ @@ -380,7 +384,8 @@ func (s *LocalScannerSuite) TestRunIncognito_QueryExceptions() { } func (s *LocalScannerSuite) TestRunIncognito_QueryExceptions_MultipleGroups() { - bundle, err := policy.BundleFromPaths("./testdata/exceptions-multiple-groups.mql.yaml") + loader := policy.DefaultBundleLoader() + bundle, err := loader.BundleFromPaths("./testdata/exceptions-multiple-groups.mql.yaml") s.Require().NoError(err) _, err = bundle.CompileExt(context.Background(), policy.BundleCompileConf{