Skip to content

Commit

Permalink
⭐️ add discovery for terraform projects for GitLab projects
Browse files Browse the repository at this point in the history
This change allows the discovery of terraform projects via the github provider. To ensure
a seamless experience, we also add support gitlab token authentication, so that the git
repo can be checked out with the same GitLab token.
  • Loading branch information
chris-rock committed Sep 17, 2023
1 parent fcc2c19 commit 121bd8e
Show file tree
Hide file tree
Showing 6 changed files with 205 additions and 15 deletions.
4 changes: 3 additions & 1 deletion apps/cnquery/cmd/builder/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -690,13 +690,15 @@ func GitlabProviderCmd(commonCmdFlags CommonFlagsFn, preRun CommonPreRunFn, runF
PreRun: func(cmd *cobra.Command, args []string) {
viper.BindPFlag("token", cmd.Flags().Lookup("token"))
viper.BindPFlag("group", cmd.Flags().Lookup("group"))
viper.BindPFlag("project", cmd.Flags().Lookup("project"))
preRun(cmd, args)
},
Run: runFn,
}
commonCmdFlags(cmd)
cmd.Flags().String("group", "", "a GitLab group to scan")
cmd.Flags().String("group", "", "a GitLab group")
cmd.MarkFlagRequired("group")
cmd.Flags().String("project", "", "a GitLab project")
cmd.Flags().String("token", "", "GitLab personal access token")
return cmd
}
Expand Down
6 changes: 6 additions & 0 deletions apps/cnquery/cmd/builder/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -640,6 +640,12 @@ func ParseTargetAsset(cmd *cobra.Command, args []string, providerType providers.
connection.Options["group"] = x
}

if x, err := cmd.Flags().GetString("project"); err != nil {
log.Fatal().Err(err).Msg("cannot parse --project value")
} else if x != "" {
connection.Options["project"] = x
}

if x, err := cmd.Flags().GetString("token"); err != nil {
log.Fatal().Err(err).Msg("cannot parse --token value")
} else if x != "" {
Expand Down
115 changes: 106 additions & 9 deletions motor/discovery/gitlab/gitlab.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,13 @@ package gitlab
import (
"context"
"errors"
"github.com/rs/zerolog/log"
terraform_resolver "go.mondoo.com/cnquery/motor/discovery/terraform"
"os"
"strings"

"github.com/xanzy/go-gitlab"
gitlab_lib "github.com/xanzy/go-gitlab"
"go.mondoo.com/cnquery/motor/asset"
"go.mondoo.com/cnquery/motor/discovery/common"
"go.mondoo.com/cnquery/motor/providers"
Expand All @@ -14,8 +19,9 @@ import (
)

const (
DiscoveryGroup = "group"
DiscoveryProject = "project"
DiscoveryGroup = "group"
DiscoveryProject = "project"
DiscoveryTerraform = "terraform"
)

type Resolver struct{}
Expand Down Expand Up @@ -58,20 +64,61 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide
case "gitlab-project":
if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProject) {
name := defaultName
project, _ := p.Project()
grp, _ := p.Group()

if name == "" {
project, _ := p.Project()
if project != nil {
name = project.NameWithNamespace
}
}

list = append(list, &asset.Asset{
projectAsset := &asset.Asset{
PlatformIds: []string{identifier},
Name: name,
Platform: pf,
Connections: []*providers.Config{pCfg}, // pass-in the current config
State: asset.State_STATE_ONLINE,
})
}

list = append(list, projectAsset)

if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryTerraform) {
terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), grp.Path, project.Path)
if err != nil {
log.Error().Err(err).Msg("error discovering terraform")
} else if len(terraformFiles) > 0 {
terraformCfg := pCfg.Clone()
terraformCfg.Backend = providers.ProviderType_TERRAFORM

terraformCfg.Options = map[string]string{
"asset-type": "hcl",
"path": "git+" + project.HTTPURLToRepo,
}

if pCfg.Credentials == nil {
token := os.Getenv("GITLAB_TOKEN")
terraformCfg.Credentials = []*vault.Credential{{
Type: vault.CredentialType_password,
User: "oauth2",
Secret: []byte(token),
}}
} else {
// add oauth2 user to the credentials
for i := range pCfg.Credentials {
cred := pCfg.Credentials[i]
if cred.Type == vault.CredentialType_password {
cred.User = "oauth2"
}
}
}

assets, err := (&terraform_resolver.Resolver{}).Resolve(ctx, projectAsset, terraformCfg, credsResolver, sfn, userIdDetectors...)
if err == nil && len(assets) > 0 {
list = append(list, assets...)
}
}
}
}
case "gitlab-group":
var grp *gitlab.Group
Expand Down Expand Up @@ -103,19 +150,69 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide
if clonedConfig.Options == nil {
clonedConfig.Options = map[string]string{}
}
clonedConfig.Options["group"] = grp.Name
clonedConfig.Options["project"] = project.Name
clonedConfig.Options["group"] = grp.Path
clonedConfig.Options["project"] = project.Path

list = append(list, &asset.Asset{
projectAsset := &asset.Asset{
PlatformIds: []string{identifier},
Name: project.NameWithNamespace,
Platform: gitlab_provider.GitLabProjectPlatform,
Connections: []*providers.Config{clonedConfig}, // pass-in the current config
State: asset.State_STATE_ONLINE,
})
}
list = append(list, projectAsset)

if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryTerraform) {
terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), grp.Path, project.Path)
if err == nil && len(terraformFiles) > 0 {
terraformCfg := pCfg.Clone()

assets, err := (&terraform_resolver.Resolver{}).Resolve(ctx, projectAsset, terraformCfg, credsResolver, sfn, userIdDetectors...)
if err == nil && len(assets) > 0 {
list = append(list, assets...)
}
}
}
}
}
}
}
return list, nil
}

// discoverTerraformHcl will check if the repository contains terraform files and return the terraform asset
func discoverTerraformHcl(ctx context.Context, client *gitlab_lib.Client, group string, project string) ([]string, error) {
opts := &gitlab_lib.ListTreeOptions{
ListOptions: gitlab_lib.ListOptions{
PerPage: 100,
},
Recursive: gitlab_lib.Bool(true),
}

nodes := []*gitlab_lib.TreeNode{}
for {
data, resp, err := client.Repositories.ListTree(group+"/"+project, opts)
if err != nil {
return nil, err
}
nodes = append(nodes, data...)

// Exit the loop when we've seen all pages.
if resp.NextPage == 0 {
break
}

// Update the page number to get the next page.
opts.Page = resp.NextPage
}

terraformFiles := []string{}
for i := range nodes {
node := nodes[i]
if node.Type == "blob" && strings.HasSuffix(node.Path, ".tf") {
terraformFiles = append(terraformFiles, node.Path)
}
}

return terraformFiles, nil
}
33 changes: 33 additions & 0 deletions motor/providers/gitlab/provider_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@
package gitlab

import (
"fmt"
"os"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
gitlab_lib "github.com/xanzy/go-gitlab"
"go.mondoo.com/cnquery/motor/providers"
)

Expand Down Expand Up @@ -42,4 +44,35 @@ func TestGitlabProject(t *testing.T) {
id, err := p.Identifier()
require.NoError(t, err)
assert.True(t, strings.HasPrefix(id, "//platformid.api.mondoo.app/runtime/gitlab/group/"))

client := p.Client()

opts := &gitlab_lib.ListTreeOptions{
ListOptions: gitlab_lib.ListOptions{
PerPage: 100,
},
Recursive: gitlab_lib.Bool(true),
}

nodes := []*gitlab_lib.TreeNode{}
for {
data, resp, err := client.Repositories.ListTree("<my-group>/<my-project>", opts)
require.NoError(t, err)
nodes = append(nodes, data...)

// Exit the loop when we've seen all pages.
if resp.NextPage == 0 {
break
}

// Update the page number to get the next page.
opts.Page = resp.NextPage
}

for i := range nodes {
node := nodes[i]
if node.Type == "blob" && strings.HasSuffix(node.Path, ".tf") {
fmt.Println(node.Path)
}
}
}
47 changes: 42 additions & 5 deletions motor/providers/terraform/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"

"io/fs"
"os"
"path/filepath"
Expand All @@ -17,6 +18,7 @@ import (
"github.com/hashicorp/hcl/v2/hclparse"
"github.com/rs/zerolog/log"
"go.mondoo.com/cnquery/motor/providers"
"go.mondoo.com/cnquery/motor/vault"
)

var (
Expand Down Expand Up @@ -44,7 +46,7 @@ func New(tc *providers.Config) (*Provider, error) {
var closer func()
path := tc.Options["path"]
if strings.HasPrefix(path, "git+https://") || strings.HasPrefix(path, "git+ssh://") {
path, closer, err = processGitForTerraform(path)
path, closer, err = processGitForTerraform(path, tc.Credentials)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -239,20 +241,55 @@ func (p *Provider) Plan() (*Plan, error) {

var reGitHttps = regexp.MustCompile(`git\+https://([^/]+)/(.*)`)

func processGitForTerraform(url string) (string, func(), error) {
// gitCloneUrl returns a git clone url from a git+https url
// If a token is provided, it will be used to clone the repo
// gitlab: git clone https://oauth2:[email protected]/vendor/package.git
func gitCloneUrl(url string, credentials []*vault.Credential) (string, error) {

user := ""
token := ""
for i := range credentials {
cred := credentials[i]
if cred.Type == vault.CredentialType_password {
user = cred.User
token = string(cred.Secret)
if token == "" && cred.Password != "" {
token = string(cred.Password)
}
}
}

m := reGitHttps.FindStringSubmatch(url)
if len(m) == 3 {
if strings.Contains(m[1], ":") {
return "", nil, errors.New("url cannot contain a port! (" + m[1] + ")")
return "", errors.New("url cannot contain a port! (" + m[1] + ")")
}

if user != "" && token != "" {
// e.g. used by GitLab
url = "https://" + user + ":" + token + "@" + m[1] + "/" + m[2]
} else if token != "" {
// e.g. used by GitHub
url = "https://" + token + "@" + m[1] + "/" + m[2]
} else {
url = "git@" + m[1] + ":" + m[2]
}
url = "git@" + m[1] + ":" + m[2]
}
// url = strings.ReplaceAll(url, "git+https://gitlab.com/", "[email protected]:")
url = strings.ReplaceAll(url, "git+ssh://", "")

if !strings.HasSuffix(url, ".git") {
url += ".git"
}
return url, nil
}

// processGitForTerraform clones a git repo and returns the path to the clone
func processGitForTerraform(url string, credentials []*vault.Credential) (string, func(), error) {
cloneUrl, err := gitCloneUrl(url, credentials)
if err != nil {
return "", nil, errors.Wrap(err, "failed to parse git clone url "+url)
}

cloneDir, err := os.MkdirTemp(os.TempDir(), "gitClone")
if err != nil {
Expand All @@ -269,7 +306,7 @@ func processGitForTerraform(url string) (string, func(), error) {

log.Info().Str("url", url).Str("path", cloneDir).Msg("git clone")
repo, err := git.PlainClone(cloneDir, false, &git.CloneOptions{
URL: url,
URL: cloneUrl,
Progress: os.Stderr,
Depth: 1,
RecurseSubmodules: git.DefaultSubmoduleRecursionDepth,
Expand Down
15 changes: 15 additions & 0 deletions motor/providers/terraform/provider_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package terraform

import (
"go.mondoo.com/cnquery/motor/vault"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -32,3 +33,17 @@ func TestModuleManifestIssue676(t *testing.T) {
require.NotNil(t, p.modulesManifest)
require.Len(t, p.modulesManifest.Records, 3)
}

func TestGitCloneUrl(t *testing.T) {
cloneUrl, err := gitCloneUrl("git+https://somegitlab.com/vendor/package.git", nil)
require.NoError(t, err)
assert.Equal(t, "[email protected]:vendor/package.git", cloneUrl)

cloneUrl, err = gitCloneUrl("git+https://somegitlab.com/vendor/package.git", []*vault.Credential{{
Type: vault.CredentialType_password,
User: "oauth2",
Password: "ACCESS_TOKEN",
}})
require.NoError(t, err)
assert.Equal(t, "https://oauth2:[email protected]/vendor/package.git", cloneUrl)
}

0 comments on commit 121bd8e

Please sign in to comment.