⭐️ gitlab terraform discovery (#1755)

This PR adds the ability to the gitlab provider to detect terraform
inside of the git repo.

```bash
cnquery shell gitlab --group example --project example-gitlab
→ loaded configuration from /Users/chris/.config/mondoo/mondoo.yml using source default
→ discover related assets for 1 asset(s)
→ resolved assets resolved-assets=2

    Available assets                               
                                                   
  > 1. example / Example Gitlab (gitlab-project)
    2. example / Example Gitlab (terraform-hcl) 
 ```

apps/cnquery/cmd/builder/common/common.go

@@ -690,13 +690,15 @@ func GitlabProviderCmd(commonCmdFlags CommonFlagsFn, preRun CommonPreRunFn, runF
 		PreRun: func(cmd *cobra.Command, args []string) {
 			viper.BindPFlag("token", cmd.Flags().Lookup("token"))
 			viper.BindPFlag("group", cmd.Flags().Lookup("group"))
+			viper.BindPFlag("project", cmd.Flags().Lookup("project"))
 			preRun(cmd, args)
 		},
 		Run: runFn,
 	}
 	commonCmdFlags(cmd)
-	cmd.Flags().String("group", "", "a GitLab group to scan")
+	cmd.Flags().String("group", "", "a GitLab group")
 	cmd.MarkFlagRequired("group")
+	cmd.Flags().String("project", "", "a GitLab project")
 	cmd.Flags().String("token", "", "GitLab personal access token")
 	return cmd
 }

apps/cnquery/cmd/builder/parse.go

@@ -640,6 +640,12 @@ func ParseTargetAsset(cmd *cobra.Command, args []string, providerType providers.
 			connection.Options["group"] = x
 		}
 
+		if x, err := cmd.Flags().GetString("project"); err != nil {
+			log.Fatal().Err(err).Msg("cannot parse --project value")
+		} else if x != "" {
+			connection.Options["project"] = x
+		}
+
 		if x, err := cmd.Flags().GetString("token"); err != nil {
 			log.Fatal().Err(err).Msg("cannot parse --token value")
 		} else if x != "" {

motor/discovery/gitlab/gitlab.go

@@ -3,8 +3,13 @@ package gitlab
 import (
 	"context"
 	"errors"
+	"github.com/rs/zerolog/log"
+	terraform_resolver "go.mondoo.com/cnquery/motor/discovery/terraform"
+	"os"
+	"strings"
 
 	"github.com/xanzy/go-gitlab"
+	gitlab_lib "github.com/xanzy/go-gitlab"
 	"go.mondoo.com/cnquery/motor/asset"
 	"go.mondoo.com/cnquery/motor/discovery/common"
 	"go.mondoo.com/cnquery/motor/providers"
@@ -14,8 +19,9 @@ import (
 )
 
 const (
-	DiscoveryGroup   = "group"
-	DiscoveryProject = "project"
+	DiscoveryGroup     = "group"
+	DiscoveryProject   = "project"
+	DiscoveryTerraform = "terraform"
 )
 
 type Resolver struct{}
@@ -58,20 +64,61 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide
 	case "gitlab-project":
 		if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProject) {
 			name := defaultName
+			project, _ := p.Project()
+			grp, _ := p.Group()
+
 			if name == "" {
-				project, _ := p.Project()
 				if project != nil {
 					name = project.NameWithNamespace
 				}
 			}
 
-			list = append(list, &asset.Asset{
+			projectAsset := &asset.Asset{
 				PlatformIds: []string{identifier},
 				Name:        name,
 				Platform:    pf,
 				Connections: []*providers.Config{pCfg}, // pass-in the current config
 				State:       asset.State_STATE_ONLINE,
-			})
+			}
+
+			list = append(list, projectAsset)
+
+			if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryTerraform) {
+				terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), grp.Path, project.Path)
+				if err != nil {
+					log.Error().Err(err).Msg("error discovering terraform")
+				} else if len(terraformFiles) > 0 {
+					terraformCfg := pCfg.Clone()
+					terraformCfg.Backend = providers.ProviderType_TERRAFORM
+
+					terraformCfg.Options = map[string]string{
+						"asset-type": "hcl",
+						"path":       "git+" + project.HTTPURLToRepo,
+					}
+
+					if pCfg.Credentials == nil {
+						token := os.Getenv("GITLAB_TOKEN")
+						terraformCfg.Credentials = []*vault.Credential{{
+							Type:   vault.CredentialType_password,
+							User:   "oauth2",
+							Secret: []byte(token),
+						}}
+					} else {
+						// add oauth2 user to the credentials
+						for i := range pCfg.Credentials {
+							cred := pCfg.Credentials[i]
+							if cred.Type == vault.CredentialType_password {
+								cred.User = "oauth2"
+							}
+						}
+					}
+
+					assets, err := (&terraform_resolver.Resolver{}).Resolve(ctx, projectAsset, terraformCfg, credsResolver, sfn, userIdDetectors...)
+					if err == nil && len(assets) > 0 {
+						list = append(list, assets...)
+					}
+				}
+			}
 		}
 	case "gitlab-group":
 		var grp *gitlab.Group
@@ -103,19 +150,69 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide
 					if clonedConfig.Options == nil {
 						clonedConfig.Options = map[string]string{}
 					}
-					clonedConfig.Options["group"] = grp.Name
-					clonedConfig.Options["project"] = project.Name
+					clonedConfig.Options["group"] = grp.Path
+					clonedConfig.Options["project"] = project.Path
 
-					list = append(list, &asset.Asset{
+					projectAsset := &asset.Asset{
 						PlatformIds: []string{identifier},
 						Name:        project.NameWithNamespace,
 						Platform:    gitlab_provider.GitLabProjectPlatform,
 						Connections: []*providers.Config{clonedConfig}, // pass-in the current config
 						State:       asset.State_STATE_ONLINE,
-					})
+					}
+					list = append(list, projectAsset)
+
+					if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryTerraform) {
+						terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), grp.Path, project.Path)
+						if err == nil && len(terraformFiles) > 0 {
+							terraformCfg := pCfg.Clone()
+
+							assets, err := (&terraform_resolver.Resolver{}).Resolve(ctx, projectAsset, terraformCfg, credsResolver, sfn, userIdDetectors...)
+							if err == nil && len(assets) > 0 {
+								list = append(list, assets...)
+							}
+						}
+					}
 				}
 			}
 		}
 	}
 	return list, nil
 }
+
+// discoverTerraformHcl will check if the repository contains terraform files and return the terraform asset
+func discoverTerraformHcl(ctx context.Context, client *gitlab_lib.Client, group string, project string) ([]string, error) {
+	opts := &gitlab_lib.ListTreeOptions{
+		ListOptions: gitlab_lib.ListOptions{
+			PerPage: 100,
+		},
+		Recursive: gitlab_lib.Bool(true),
+	}
+
+	nodes := []*gitlab_lib.TreeNode{}
+	for {
+		data, resp, err := client.Repositories.ListTree(group+"/"+project, opts)
+		if err != nil {
+			return nil, err
+		}
+		nodes = append(nodes, data...)
+
+		// Exit the loop when we've seen all pages.
+		if resp.NextPage == 0 {
+			break
+		}
+
+		// Update the page number to get the next page.
+		opts.Page = resp.NextPage
+	}
+
+	terraformFiles := []string{}
+	for i := range nodes {
+		node := nodes[i]
+		if node.Type == "blob" && strings.HasSuffix(node.Path, ".tf") {
+			terraformFiles = append(terraformFiles, node.Path)
+		}
+	}
+
+	return terraformFiles, nil
+}

motor/providers/terraform/provider.go

@@ -5,6 +5,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -17,6 +18,7 @@ import (
 	"github.com/hashicorp/hcl/v2/hclparse"
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/motor/providers"
+	"go.mondoo.com/cnquery/motor/vault"
 )
 
 var (
@@ -44,7 +46,7 @@ func New(tc *providers.Config) (*Provider, error) {
 	var closer func()
 	path := tc.Options["path"]
 	if strings.HasPrefix(path, "git+https://") || strings.HasPrefix(path, "git+ssh://") {
-		path, closer, err = processGitForTerraform(path)
+		path, closer, err = processGitForTerraform(path, tc.Credentials)
 		if err != nil {
 			return nil, err
 		}
@@ -239,20 +241,55 @@ func (p *Provider) Plan() (*Plan, error) {
 
 var reGitHttps = regexp.MustCompile(`git\+https://([^/]+)/(.*)`)
 
-func processGitForTerraform(url string) (string, func(), error) {
+// gitCloneUrl returns a git clone url from a git+https url
+// If a token is provided, it will be used to clone the repo
+// gitlab: git clone https://oauth2:[email protected]/vendor/package.git
+func gitCloneUrl(url string, credentials []*vault.Credential) (string, error) {
+
+	user := ""
+	token := ""
+	for i := range credentials {
+		cred := credentials[i]
+		if cred.Type == vault.CredentialType_password {
+			user = cred.User
+			token = string(cred.Secret)
+			if token == "" && cred.Password != "" {
+				token = string(cred.Password)
+			}
+		}
+	}
+
 	m := reGitHttps.FindStringSubmatch(url)
 	if len(m) == 3 {
 		if strings.Contains(m[1], ":") {
-			return "", nil, errors.New("url cannot contain a port! (" + m[1] + ")")
+			return "", errors.New("url cannot contain a port! (" + m[1] + ")")
+		}
+
+		if user != "" && token != "" {
+			// e.g. used by GitLab
+			url = "https://" + user + ":" + token + "@" + m[1] + "/" + m[2]
+		} else if token != "" {
+			// e.g. used by GitHub
+			url = "https://" + token + "@" + m[1] + "/" + m[2]
+		} else {
+			url = "git@" + m[1] + ":" + m[2]
 		}
-		url = "git@" + m[1] + ":" + m[2]
 	}
 	// url = strings.ReplaceAll(url, "git+https://gitlab.com/", "[email protected]:")
 	url = strings.ReplaceAll(url, "git+ssh://", "")
 
 	if !strings.HasSuffix(url, ".git") {
 		url += ".git"
 	}
+	return url, nil
+}
+
+// processGitForTerraform clones a git repo and returns the path to the clone
+func processGitForTerraform(url string, credentials []*vault.Credential) (string, func(), error) {
+	cloneUrl, err := gitCloneUrl(url, credentials)
+	if err != nil {
+		return "", nil, errors.Wrap(err, "failed to parse git clone url "+url)
+	}
 
 	cloneDir, err := os.MkdirTemp(os.TempDir(), "gitClone")
 	if err != nil {
@@ -269,7 +306,7 @@ func processGitForTerraform(url string) (string, func(), error) {
 
 	log.Info().Str("url", url).Str("path", cloneDir).Msg("git clone")
 	repo, err := git.PlainClone(cloneDir, false, &git.CloneOptions{
-		URL:               url,
+		URL:               cloneUrl,
 		Progress:          os.Stderr,
 		Depth:             1,
 		RecurseSubmodules: git.DefaultSubmoduleRecursionDepth,

motor/providers/terraform/provider_test.go

@@ -1,6 +1,7 @@
 package terraform
 
 import (
+	"go.mondoo.com/cnquery/motor/vault"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -32,3 +33,17 @@ func TestModuleManifestIssue676(t *testing.T) {
 	require.NotNil(t, p.modulesManifest)
 	require.Len(t, p.modulesManifest.Records, 3)
 }
+
+func TestGitCloneUrl(t *testing.T) {
+	cloneUrl, err := gitCloneUrl("git+https://somegitlab.com/vendor/package.git", nil)
+	require.NoError(t, err)
+	assert.Equal(t, "[email protected]:vendor/package.git", cloneUrl)
+
+	cloneUrl, err = gitCloneUrl("git+https://somegitlab.com/vendor/package.git", []*vault.Credential{{
+		Type:     vault.CredentialType_password,
+		User:     "oauth2",
+		Password: "ACCESS_TOKEN",
+	}})
+	require.NoError(t, err)
+	assert.Equal(t, "https://oauth2:[email protected]/vendor/package.git", cloneUrl)
+}