From 6a0d9ad1abe9cb77f729c50c9716a538196d3b3e Mon Sep 17 00:00:00 2001 From: Christoph Hartmann Date: Sun, 17 Sep 2023 13:21:53 -0700 Subject: [PATCH] =?UTF-8?q?=E2=AD=90=EF=B8=8F=20add=20discovery=20for=20te?= =?UTF-8?q?rraform=20projects=20for=20GitLab=20projects?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change allows the discovery of terraform projects via the github provider. To ensure a seamless experience, we also add support gitlab token authentication, so that the git repo can be checked out with the same GitLab token. --- apps/cnquery/cmd/builder/common/common.go | 4 +- apps/cnquery/cmd/builder/parse.go | 6 ++ motor/discovery/gitlab/gitlab.go | 115 +++++++++++++++++++-- motor/providers/terraform/provider.go | 47 ++++++++- motor/providers/terraform/provider_test.go | 15 +++ 5 files changed, 172 insertions(+), 15 deletions(-) diff --git a/apps/cnquery/cmd/builder/common/common.go b/apps/cnquery/cmd/builder/common/common.go index 1167b9fa9f..13899fe9ee 100644 --- a/apps/cnquery/cmd/builder/common/common.go +++ b/apps/cnquery/cmd/builder/common/common.go @@ -690,13 +690,15 @@ func GitlabProviderCmd(commonCmdFlags CommonFlagsFn, preRun CommonPreRunFn, runF PreRun: func(cmd *cobra.Command, args []string) { viper.BindPFlag("token", cmd.Flags().Lookup("token")) viper.BindPFlag("group", cmd.Flags().Lookup("group")) + viper.BindPFlag("project", cmd.Flags().Lookup("project")) preRun(cmd, args) }, Run: runFn, } commonCmdFlags(cmd) - cmd.Flags().String("group", "", "a GitLab group to scan") + cmd.Flags().String("group", "", "a GitLab group") cmd.MarkFlagRequired("group") + cmd.Flags().String("project", "", "a GitLab project") cmd.Flags().String("token", "", "GitLab personal access token") return cmd } diff --git a/apps/cnquery/cmd/builder/parse.go b/apps/cnquery/cmd/builder/parse.go index 09937d1bf2..7d4ddc6017 100644 --- a/apps/cnquery/cmd/builder/parse.go +++ b/apps/cnquery/cmd/builder/parse.go @@ -640,6 +640,12 @@ func ParseTargetAsset(cmd *cobra.Command, args []string, providerType providers. connection.Options["group"] = x } + if x, err := cmd.Flags().GetString("project"); err != nil { + log.Fatal().Err(err).Msg("cannot parse --project value") + } else if x != "" { + connection.Options["project"] = x + } + if x, err := cmd.Flags().GetString("token"); err != nil { log.Fatal().Err(err).Msg("cannot parse --token value") } else if x != "" { diff --git a/motor/discovery/gitlab/gitlab.go b/motor/discovery/gitlab/gitlab.go index f599c2b689..de754f94f4 100644 --- a/motor/discovery/gitlab/gitlab.go +++ b/motor/discovery/gitlab/gitlab.go @@ -3,8 +3,13 @@ package gitlab import ( "context" "errors" + "github.com/rs/zerolog/log" + terraform_resolver "go.mondoo.com/cnquery/motor/discovery/terraform" + "os" + "strings" "github.com/xanzy/go-gitlab" + gitlab_lib "github.com/xanzy/go-gitlab" "go.mondoo.com/cnquery/motor/asset" "go.mondoo.com/cnquery/motor/discovery/common" "go.mondoo.com/cnquery/motor/providers" @@ -14,8 +19,9 @@ import ( ) const ( - DiscoveryGroup = "group" - DiscoveryProject = "project" + DiscoveryGroup = "group" + DiscoveryProject = "project" + DiscoveryTerraform = "terraform" ) type Resolver struct{} @@ -58,20 +64,61 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide case "gitlab-project": if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProject) { name := defaultName + project, _ := p.Project() + grp, _ := p.Group() + if name == "" { - project, _ := p.Project() if project != nil { name = project.NameWithNamespace } } - list = append(list, &asset.Asset{ + projectAsset := &asset.Asset{ PlatformIds: []string{identifier}, Name: name, Platform: pf, Connections: []*providers.Config{pCfg}, // pass-in the current config State: asset.State_STATE_ONLINE, - }) + } + + list = append(list, projectAsset) + + if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryTerraform) { + terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), grp.Path, project.Path) + if err != nil { + log.Error().Err(err).Msg("error discovering terraform") + } else if len(terraformFiles) > 0 { + terraformCfg := pCfg.Clone() + terraformCfg.Backend = providers.ProviderType_TERRAFORM + + terraformCfg.Options = map[string]string{ + "asset-type": "hcl", + "path": "git+" + project.HTTPURLToRepo, + } + + if pCfg.Credentials == nil { + token := os.Getenv("GITLAB_TOKEN") + terraformCfg.Credentials = []*vault.Credential{{ + Type: vault.CredentialType_password, + User: "oauth2", + Secret: []byte(token), + }} + } else { + // add oauth2 user to the credentials + for i := range pCfg.Credentials { + cred := pCfg.Credentials[i] + if cred.Type == vault.CredentialType_password { + cred.User = "oauth2" + } + } + } + + assets, err := (&terraform_resolver.Resolver{}).Resolve(ctx, projectAsset, terraformCfg, credsResolver, sfn, userIdDetectors...) + if err == nil && len(assets) > 0 { + list = append(list, assets...) + } + } + } } case "gitlab-group": var grp *gitlab.Group @@ -103,19 +150,69 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide if clonedConfig.Options == nil { clonedConfig.Options = map[string]string{} } - clonedConfig.Options["group"] = grp.Name - clonedConfig.Options["project"] = project.Name + clonedConfig.Options["group"] = grp.Path + clonedConfig.Options["project"] = project.Path - list = append(list, &asset.Asset{ + projectAsset := &asset.Asset{ PlatformIds: []string{identifier}, Name: project.NameWithNamespace, Platform: gitlab_provider.GitLabProjectPlatform, Connections: []*providers.Config{clonedConfig}, // pass-in the current config State: asset.State_STATE_ONLINE, - }) + } + list = append(list, projectAsset) + + if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryTerraform) { + terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), grp.Path, project.Path) + if err == nil && len(terraformFiles) > 0 { + terraformCfg := pCfg.Clone() + + assets, err := (&terraform_resolver.Resolver{}).Resolve(ctx, projectAsset, terraformCfg, credsResolver, sfn, userIdDetectors...) + if err == nil && len(assets) > 0 { + list = append(list, assets...) + } + } + } } } } } return list, nil } + +// discoverTerraformHcl will check if the repository contains terraform files and return the terraform asset +func discoverTerraformHcl(ctx context.Context, client *gitlab_lib.Client, group string, project string) ([]string, error) { + opts := &gitlab_lib.ListTreeOptions{ + ListOptions: gitlab_lib.ListOptions{ + PerPage: 100, + }, + Recursive: gitlab_lib.Bool(true), + } + + nodes := []*gitlab_lib.TreeNode{} + for { + data, resp, err := client.Repositories.ListTree(group+"/"+project, opts) + if err != nil { + return nil, err + } + nodes = append(nodes, data...) + + // Exit the loop when we've seen all pages. + if resp.NextPage == 0 { + break + } + + // Update the page number to get the next page. + opts.Page = resp.NextPage + } + + terraformFiles := []string{} + for i := range nodes { + node := nodes[i] + if node.Type == "blob" && strings.HasSuffix(node.Path, ".tf") { + terraformFiles = append(terraformFiles, node.Path) + } + } + + return terraformFiles, nil +} diff --git a/motor/providers/terraform/provider.go b/motor/providers/terraform/provider.go index 058dda96fd..85cfe0197b 100644 --- a/motor/providers/terraform/provider.go +++ b/motor/providers/terraform/provider.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "io/fs" "os" "path/filepath" @@ -17,6 +18,7 @@ import ( "github.com/hashicorp/hcl/v2/hclparse" "github.com/rs/zerolog/log" "go.mondoo.com/cnquery/motor/providers" + "go.mondoo.com/cnquery/motor/vault" ) var ( @@ -44,7 +46,7 @@ func New(tc *providers.Config) (*Provider, error) { var closer func() path := tc.Options["path"] if strings.HasPrefix(path, "git+https://") || strings.HasPrefix(path, "git+ssh://") { - path, closer, err = processGitForTerraform(path) + path, closer, err = processGitForTerraform(path, tc.Credentials) if err != nil { return nil, err } @@ -239,13 +241,39 @@ func (p *Provider) Plan() (*Plan, error) { var reGitHttps = regexp.MustCompile(`git\+https://([^/]+)/(.*)`) -func processGitForTerraform(url string) (string, func(), error) { +// gitCloneUrl returns a git clone url from a git+https url +// If a token is provided, it will be used to clone the repo +// gitlab: git clone https://oauth2:ACCESS_TOKEN@somegitlab.com/vendor/package.git +func gitCloneUrl(url string, credentials []*vault.Credential) (string, error) { + + user := "" + token := "" + for i := range credentials { + cred := credentials[i] + if cred.Type == vault.CredentialType_password { + user = cred.User + token = string(cred.Secret) + if token == "" && cred.Password != "" { + token = string(cred.Password) + } + } + } + m := reGitHttps.FindStringSubmatch(url) if len(m) == 3 { if strings.Contains(m[1], ":") { - return "", nil, errors.New("url cannot contain a port! (" + m[1] + ")") + return "", errors.New("url cannot contain a port! (" + m[1] + ")") + } + + if user != "" && token != "" { + // e.g. used by GitLab + url = "https://" + user + ":" + token + "@" + m[1] + "/" + m[2] + } else if token != "" { + // e.g. used by GitHub + url = "https://" + token + "@" + m[1] + "/" + m[2] + } else { + url = "git@" + m[1] + ":" + m[2] } - url = "git@" + m[1] + ":" + m[2] } // url = strings.ReplaceAll(url, "git+https://gitlab.com/", "git@gitlab.com:") url = strings.ReplaceAll(url, "git+ssh://", "") @@ -253,6 +281,15 @@ func processGitForTerraform(url string) (string, func(), error) { if !strings.HasSuffix(url, ".git") { url += ".git" } + return url, nil +} + +// processGitForTerraform clones a git repo and returns the path to the clone +func processGitForTerraform(url string, credentials []*vault.Credential) (string, func(), error) { + cloneUrl, err := gitCloneUrl(url, credentials) + if err != nil { + return "", nil, errors.Wrap(err, "failed to parse git clone url "+url) + } cloneDir, err := os.MkdirTemp(os.TempDir(), "gitClone") if err != nil { @@ -269,7 +306,7 @@ func processGitForTerraform(url string) (string, func(), error) { log.Info().Str("url", url).Str("path", cloneDir).Msg("git clone") repo, err := git.PlainClone(cloneDir, false, &git.CloneOptions{ - URL: url, + URL: cloneUrl, Progress: os.Stderr, Depth: 1, RecurseSubmodules: git.DefaultSubmoduleRecursionDepth, diff --git a/motor/providers/terraform/provider_test.go b/motor/providers/terraform/provider_test.go index 5f0d197f5a..99c984a13f 100644 --- a/motor/providers/terraform/provider_test.go +++ b/motor/providers/terraform/provider_test.go @@ -1,6 +1,7 @@ package terraform import ( + "go.mondoo.com/cnquery/motor/vault" "testing" "github.com/stretchr/testify/assert" @@ -32,3 +33,17 @@ func TestModuleManifestIssue676(t *testing.T) { require.NotNil(t, p.modulesManifest) require.Len(t, p.modulesManifest.Records, 3) } + +func TestGitCloneUrl(t *testing.T) { + cloneUrl, err := gitCloneUrl("git+https://somegitlab.com/vendor/package.git", nil) + require.NoError(t, err) + assert.Equal(t, "git@somegitlab.com:vendor/package.git", cloneUrl) + + cloneUrl, err = gitCloneUrl("git+https://somegitlab.com/vendor/package.git", []*vault.Credential{{ + Type: vault.CredentialType_password, + User: "oauth2", + Password: "ACCESS_TOKEN", + }}) + require.NoError(t, err) + assert.Equal(t, "https://oauth2:ACCESS_TOKEN@somegitlab.com/vendor/package.git", cloneUrl) +}