From 70ccc76b56018c0de25459dc307b37ea88e54346 Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Mon, 27 May 2024 16:57:44 +0200 Subject: [PATCH 1/2] =?UTF-8?q?=E2=AD=90=EF=B8=8F=20github=20terraform=20d?= =?UTF-8?q?iscovery?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ivan Milchev --- providers/github/connection/platform.go | 2 +- providers/github/resources/discovery.go | 80 +++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/providers/github/connection/platform.go b/providers/github/connection/platform.go index 67af86d5aa..0097a18a4c 100644 --- a/providers/github/connection/platform.go +++ b/providers/github/connection/platform.go @@ -16,6 +16,7 @@ const ( DiscoveryRepository = "repository" // deprecated: use repos DiscoveryUser = "user" // deprecated: use users DiscoveryOrganization = "organization" + DiscoveryTerraform = "terraform" ) var ( @@ -62,7 +63,6 @@ func (c *GithubConnection) PlatformInfo() (*inventory.Platform, error) { } if userId := conf.Options["user"]; userId != "" { - return NewGithubUserPlatform(userId), nil } diff --git a/providers/github/resources/discovery.go b/providers/github/resources/discovery.go index 764eb1e20f..00efbc927c 100644 --- a/providers/github/resources/discovery.go +++ b/providers/github/resources/discovery.go @@ -7,12 +7,15 @@ import ( "strings" "github.com/gobwas/glob" + "github.com/google/go-github/v61/github" "github.com/rs/zerolog/log" "go.mondoo.com/cnquery/v11/llx" "go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory" "go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin" + "go.mondoo.com/cnquery/v11/providers-sdk/v1/vault" "go.mondoo.com/cnquery/v11/providers/github/connection" "go.mondoo.com/cnquery/v11/utils/stringx" + "google.golang.org/protobuf/proto" ) func Discover(runtime *plugin.Runtime, opts map[string]string) (*inventory.Inventory, error) { @@ -116,6 +119,14 @@ func org(runtime *plugin.Runtime, orgName string, conn *connection.GithubConnect Labels: make(map[string]string), Connections: []*inventory.Config{cfg}, }) + + if stringx.ContainsAnyOf(targets, connection.DiscoveryAll, connection.DiscoveryTerraform) { + terraformAssets, err := discoverTerraform(conn, repo) + if err != nil { + return nil, err + } + assetList = append(assetList, terraformAssets...) + } } } if stringx.ContainsAnyOf(targets, connection.DiscoveryUsers, connection.DiscoveryUser) { @@ -165,6 +176,14 @@ func repo(runtime *plugin.Runtime, repoName string, owner string, conn *connecti Connections: []*inventory.Config{cfg}, }) + if stringx.ContainsAnyOf(targets, connection.DiscoveryAll, connection.DiscoveryTerraform) { + terraformAssets, err := discoverTerraform(conn, repo) + if err != nil { + return nil, err + } + assetList = append(assetList, terraformAssets...) + } + return assetList, nil } @@ -257,3 +276,64 @@ func (f *ReposFilter) skipRepo(namespace string) bool { return false } + +func discoverTerraform(conn *connection.GithubConnection, repo *mqlGithubRepository) ([]*inventory.Asset, error) { + // For git clone we need to set the user to oauth2 to be usable with the token. + conf := conn.Asset().Connections[0] + creds := make([]*vault.Credential, len(conf.Credentials)) + for i := range conf.Credentials { + cred := conf.Credentials[i] + cc := proto.Clone(cred).(*vault.Credential) + if cc.User == "" { + cc.User = "oauth2" + } + creds[i] = cc + } + + files := repo.GetFiles() + if files.Error != nil { + return nil, files.Error + } + + var res []*inventory.Asset + hasTf, err := hasTerraformHcl(conn.Client(), files) + if err != nil { + log.Error().Err(err).Str("project", repo.FullName.Data).Msg("failed to discover terraform repo in gitlab") + } else if hasTf { + res = append(res, &inventory.Asset{ + Connections: []*inventory.Config{{ + Type: "terraform-hcl-git", + Options: map[string]string{ + "ssh-url": repo.SshUrl.Data, + "http-url": repo.CloneUrl.Data, + }, + Credentials: creds, + }}, + }) + } + return res, nil +} + +// hasTerraformHcl will check if the repository contains terraform files +func hasTerraformHcl(client *github.Client, files *plugin.TValue[[]interface{}]) (bool, error) { + for _, f := range files.Data { + file := f.(*mqlGithubFile) + children := file.GetFiles() + if children.Error != nil { + return false, children.Error + } + hasTf, err := hasTerraformHcl(client, children) + if err != nil { + return false, err + } + if hasTf { + return true, nil + } + + if strings.HasSuffix(file.Path.Data, ".tf") { + return true, nil + } + } + + return false, nil +} From 0019c186559edecd10ab1a48df68808f5b8231e0 Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Mon, 27 May 2024 18:44:18 +0200 Subject: [PATCH 2/2] use github search api when looking for terraform files Signed-off-by: Ivan Milchev --- providers/github/resources/discovery.go | 34 ++++++------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/providers/github/resources/discovery.go b/providers/github/resources/discovery.go index 00efbc927c..315fd024c6 100644 --- a/providers/github/resources/discovery.go +++ b/providers/github/resources/discovery.go @@ -4,6 +4,7 @@ package resources import ( + "context" "strings" "github.com/gobwas/glob" @@ -290,13 +291,8 @@ func discoverTerraform(conn *connection.GithubConnection, repo *mqlGithubReposit creds[i] = cc } - files := repo.GetFiles() - if files.Error != nil { - return nil, files.Error - } - var res []*inventory.Asset - hasTf, err := hasTerraformHcl(conn.Client(), files) + hasTf, err := hasTerraformHcl(conn.Client(), repo) if err != nil { log.Error().Err(err).Str("project", repo.FullName.Data).Msg("failed to discover terraform repo in gitlab") } else if hasTf { @@ -315,25 +311,11 @@ func discoverTerraform(conn *connection.GithubConnection, repo *mqlGithubReposit } // hasTerraformHcl will check if the repository contains terraform files -func hasTerraformHcl(client *github.Client, files *plugin.TValue[[]interface{}]) (bool, error) { - for _, f := range files.Data { - file := f.(*mqlGithubFile) - children := file.GetFiles() - if children.Error != nil { - return false, children.Error - } - hasTf, err := hasTerraformHcl(client, children) - if err != nil { - return false, err - } - if hasTf { - return true, nil - } - - if strings.HasSuffix(file.Path.Data, ".tf") { - return true, nil - } +func hasTerraformHcl(client *github.Client, repo *mqlGithubRepository) (bool, error) { + query := "repo:" + repo.FullName.Data + " extension:tf" + res, _, err := client.Search.Code(context.Background(), query, &github.SearchOptions{}) + if err != nil { + return false, err } - - return false, nil + return res.GetTotal() > 0, nil }