⭐ support terraform discovery over gitlab

providers/gitlab/go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/xanzy/go-gitlab v0.91.1
 	go.mondoo.com/cnquery v0.0.0-20230920205842-55a158611de3
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
+	google.golang.org/protobuf v1.31.0
 )
 
 require (
@@ -62,7 +63,6 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
 	google.golang.org/grpc v1.58.1 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

providers/gitlab/provider/discovery.go

@@ -10,8 +10,10 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/xanzy/go-gitlab"
 	"go.mondoo.com/cnquery/providers-sdk/v1/inventory"
+	"go.mondoo.com/cnquery/providers-sdk/v1/vault"
 	"go.mondoo.com/cnquery/providers/gitlab/connection"
 	"golang.org/x/exp/slices"
+	"google.golang.org/protobuf/proto"
 )
 
 func (s *Service) discover(root *inventory.Asset, conn *connection.GitLabConnection) (*inventory.Inventory, error) {
@@ -159,6 +161,17 @@ func discoverGroupProjects(conn *connection.GitLabConnection, gid interface{}) (
 }
 
 func (s *Service) discoverTerraform(root *inventory.Asset, conn *connection.GitLabConnection, projects []*gitlab.Project) ([]*inventory.Asset, error) {
+	// For git clone we need to set the user to oauth2 to be usable with the token.
+	creds := make([]*vault.Credential, len(conn.Conf.Credentials))
+	for i := range conn.Conf.Credentials {
+		cred := conn.Conf.Credentials[i]
+		cc := proto.Clone(cred).(*vault.Credential)
+		if cc.User == "" {
+			cc.User = "oauth2"
+		}
+		creds[i] = cc
+	}
+
 	var res []*inventory.Asset
 	for i := range projects {
 		project := projects[i]
@@ -173,7 +186,7 @@ func (s *Service) discoverTerraform(root *inventory.Asset, conn *connection.GitL
 						"ssh-url":  project.SSHURLToRepo,
 						"http-url": project.HTTPURLToRepo,
 					},
-					Credentials: conn.Conf.Credentials,
+					Credentials: creds,
 				}},
 			})
 		}

providers/terraform/config/config.go

@@ -16,6 +16,7 @@ var Config = plugin.Provider{
 		provider.StateConnectionType,
 		provider.PlanConnectionType,
 		provider.HclConnectionType,
+		provider.HclGitConnectionType,
 	},
 	Connectors: []plugin.Connector{
 		{

providers/terraform/connection/connection.go

@@ -25,9 +25,14 @@ type Connection struct {
 	modulesManifest *ModuleManifest
 	state           *State
 	plan            *Plan
+	closer          func()
 }
 
-func (c *Connection) Close() {}
+func (c *Connection) Close() {
+	if c.closer != nil {
+		c.closer()
+	}
+}
 
 func (c *Connection) Kind() string {
 	return "code"

providers/terraform/connection/hcl_manifest.go

@@ -7,15 +7,18 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/fs"
+	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 
+	"github.com/go-git/go-git/v5"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/providers-sdk/v1/inventory"
+	"go.mondoo.com/cnquery/providers-sdk/v1/vault"
 )
 
 func ParseTerraformModuleManifest(manifestPath string) (*ModuleManifest, error) {
@@ -42,7 +45,11 @@ var MODULE_EXAMPLES = regexp.MustCompile(`^.*/modules/.+/examples/.+`)
 
 func NewHclConnection(id uint32, asset *inventory.Asset) (*Connection, error) {
 	cc := asset.Connections[0]
+	path := cc.Options["path"]
+	return newHclConnection(path, asset)
+}
 
+func newHclConnection(path string, asset *inventory.Asset) (*Connection, error) {
 	// NOTE: right now we are only supporting to load either state, plan or hcl files but not at the same time
 
 	var assetType terraformAssetType
@@ -52,7 +59,6 @@ func NewHclConnection(id uint32, asset *inventory.Asset) (*Connection, error) {
 	var modulesManifest *ModuleManifest
 
 	assetType = configurationfiles
-	path := cc.Options["path"]
 	// FIXME: cannot handle relative paths
 	stat, err := os.Stat(path)
 	if os.IsNotExist(err) {
@@ -121,3 +127,94 @@ func NewHclConnection(id uint32, asset *inventory.Asset) (*Connection, error) {
 		modulesManifest: modulesManifest,
 	}, nil
 }
+
+func NewHclGitConnection(id uint32, asset *inventory.Asset) (*Connection, error) {
+	cc := asset.Connections[0]
+
+	if len(cc.Options) == 0 {
+		return nil, errors.New("missing URLs in options for HCL over Git connection")
+	}
+
+	user := ""
+	token := ""
+	for i := range cc.Credentials {
+		cred := cc.Credentials[i]
+		if cred.Type == vault.CredentialType_password {
+			user = cred.User
+			token = string(cred.Secret)
+			if token == "" && cred.Password != "" {
+				token = string(cred.Password)
+			}
+		}
+	}
+
+	gitUrl := ""
+
+	// If a token is provided, it will be used to clone the repo
+	// gitlab: git clone https://oauth2:[email protected]/vendor/package.git
+	// if sshUrl := cc.Options["ssh-url"]; sshUrl != "" { ... not doing ssh url right now
+	if httpUrl := cc.Options["http-url"]; httpUrl != "" {
+		u, err := url.Parse(httpUrl)
+		if err != nil {
+			return nil, errors.New("failed to parse url for git repo: " + httpUrl)
+		}
+
+		if user != "" && token != "" {
+			u.User = url.UserPassword(user, token)
+		} else if token != "" {
+			u.User = url.User(token)
+		}
+
+		gitUrl = u.String()
+	}
+
+	if gitUrl == "" {
+		return nil, errors.New("missing url for git repo " + asset.Name)
+	}
+
+	path, closer, err := gitClone(gitUrl)
+	if err != nil {
+		return nil, err
+	}
+	conn, err := newHclConnection(path, asset)
+	if err != nil {
+		return nil, err
+	}
+	conn.closer = closer
+	return conn, nil
+}
+
+func gitClone(url string) (string, func(), error) {
+	cloneDir, err := os.MkdirTemp(os.TempDir(), "gitClone")
+	if err != nil {
+		return "", nil, errors.Wrap(err, "failed to create temporary dir for git processing")
+	}
+
+	closer := func() {
+		log.Info().Str("path", cloneDir).Msg("cleaning up git clone")
+		if err = os.RemoveAll(cloneDir); err != nil {
+			log.Error().Err(err).Msg("failed to remove temporary dir for git processing")
+		}
+	}
+
+	log.Info().Str("url", url).Str("path", cloneDir).Msg("git clone")
+	repo, err := git.PlainClone(cloneDir, false, &git.CloneOptions{
+		URL:               url,
+		Progress:          os.Stderr,
+		Depth:             1,
+		RecurseSubmodules: git.DefaultSubmoduleRecursionDepth,
+	})
+	if err != nil {
+		closer()
+		return "", nil, errors.Wrap(err, "failed to clone git repo "+url)
+	}
+
+	ref, err := repo.Head()
+	if err != nil {
+		closer()
+		return "", nil, errors.Wrap(err, "failed to get head of git repo "+url)
+	}
+	log.Info().Str("url", url).Str("path", cloneDir).Str("head", ref.Hash().String()).Msg("finshed git clone")
+
+	return cloneDir, closer, nil
+}

providers/terraform/go.mod

@@ -4,29 +4,44 @@ replace go.mondoo.com/cnquery => ../..
 
 go 1.20
 
-require github.com/hashicorp/hcl/v2 v2.18.0
+require (
+	github.com/go-git/go-git/v5 v5.9.0
+	github.com/hashicorp/hcl/v2 v2.18.0
+)
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/cockroachdb/errors v1.11.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/getsentry/sentry-go v0.24.1 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/gofrs/uuid v4.4.0+incompatible // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/uuid v1.3.1 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-plugin v1.5.1 // indirect
 	github.com/hashicorp/yamux v0.1.1 // indirect
 	github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
@@ -36,21 +51,28 @@ require (
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/muesli/termenv v0.15.2 // indirect
 	github.com/oklog/run v1.1.0 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/segmentio/fasthash v1.0.3 // indirect
 	github.com/segmentio/ksuid v1.0.4 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/skeema/knownhosts v1.2.0 // indirect
 	github.com/smarty/assertions v1.15.1 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	go.mondoo.com/ranger-rpc v0.5.1 // indirect
 	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.15.0 // indirect
 	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
 	google.golang.org/grpc v1.58.1 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	moul.io/http2curl v1.0.0 // indirect