diff --git a/explorer/scan/discovery.go b/explorer/scan/discovery.go index 73bed4b9d6..6be27a1bc8 100644 --- a/explorer/scan/discovery.go +++ b/explorer/scan/discovery.go @@ -151,6 +151,12 @@ func DiscoverAssets(ctx context.Context, inv *inventory.Inventory, upstream *ups } func discoverAssets(rootAssetWithRuntime *AssetWithRuntime, resolvedRootAsset *inventory.Asset, discoveredAssets *DiscoveredAssets, runtimeLabels map[string]string, upstream *upstream.UpstreamConfig, recording llx.Recording) { + // It is possible that we did not discover any assets under the root asset. In that case the inventory + // would be nil and we can return + if rootAssetWithRuntime.Runtime.Provider.Connection.Inventory == nil { + return + } + // for all discovered assets, we apply mondoo-specific labels and annotations that come from the root asset for _, a := range rootAssetWithRuntime.Runtime.Provider.Connection.Inventory.Spec.Assets { // create runtime for root asset diff --git a/providers/github/resources/discovery.go b/providers/github/resources/discovery.go index a5eac2f502..effc549e60 100644 --- a/providers/github/resources/discovery.go +++ b/providers/github/resources/discovery.go @@ -336,7 +336,26 @@ func hasTerraformHcl(client *github.Client, repo *mqlGithubRepository) (bool, er if err != nil { return false, err } - return res.GetTotal() > 0, nil + + // Ignore tf files that are hidden or are in a hidden folder + nonHiddenTf := 0 + for _, code := range res.CodeResults { + fragments := strings.Split(code.GetPath(), "/") + // skip hidden files + isHidden := false + for _, fragment := range fragments { + if strings.HasPrefix(fragment, ".") { + isHidden = true + break + } + } + + if !isHidden { + nonHiddenTf++ + } + } + + return nonHiddenTf > 0, nil } func discoverK8sManifests(conn *connection.GithubConnection, repo *mqlGithubRepository) ([]*inventory.Asset, error) { @@ -383,6 +402,13 @@ func hasYaml(client *github.Client, repo *mqlGithubRepository) (bool, error) { // Ignore YAML files that are hidden or are in a hidden folder nonHiddenYaml := 0 for _, code := range res.CodeResults { + path := code.GetPath() + + // Skip MQL files + if strings.HasSuffix(path, "mql.yaml") || strings.HasSuffix(path, "mql.yml") { + continue + } + fragments := strings.Split(code.GetPath(), "/") // skip hidden files isHidden := false diff --git a/providers/gitlab/provider/discovery.go b/providers/gitlab/provider/discovery.go index 4fc7598720..ab5198633b 100644 --- a/providers/gitlab/provider/discovery.go +++ b/providers/gitlab/provider/discovery.go @@ -77,8 +77,8 @@ func (s *Service) discover(root *inventory.Asset, conn *connection.GitLabConnect } } - if slices.Contains(targets, DiscoveryTerraform) { - repos, err := s.discoverTerraform(root, conn, projects) + if slices.Contains(targets, DiscoveryTerraform) || slices.Contains(targets, DiscoveryK8sManifests) { + repos, err := s.discoverTypes(targets, conn, projects) if err != nil { return nil, err } @@ -279,7 +279,11 @@ func listAllGroups(conn *connection.GitLabConnection) ([]*gitlab.Group, error) { return groups, nil } -func (s *Service) discoverTerraform(root *inventory.Asset, conn *connection.GitLabConnection, projects []*gitlab.Project) ([]*inventory.Asset, error) { +func (s *Service) discoverTypes(targets []string, conn *connection.GitLabConnection, projects []*gitlab.Project) ([]*inventory.Asset, error) { + if !slices.Contains(targets, DiscoveryTerraform) && !slices.Contains(targets, DiscoveryK8sManifests) { + return nil, nil + } + // For git clone we need to set the user to oauth2 to be usable with the token. creds := make([]*vault.Credential, len(conn.Conf.Credentials)) for i := range conn.Conf.Credentials { @@ -294,10 +298,13 @@ func (s *Service) discoverTerraform(root *inventory.Asset, conn *connection.GitL var res []*inventory.Asset for i := range projects { project := projects[i] - files, err := discoverTerraformHcl(conn.Client(), project.ID) + discoveredTypes, err := discoverRepoTypes(conn.Client(), project.ID) if err != nil { log.Error().Err(err).Str("project", project.PathWithNamespace).Msg("failed to discover terraform repo in gitlab") - } else if len(files) != 0 { + continue + } + + if discoveredTypes.terraform && slices.Contains(targets, DiscoveryTerraform) { res = append(res, &inventory.Asset{ Connections: []*inventory.Config{{ Type: "terraform-hcl-git", @@ -309,17 +316,36 @@ func (s *Service) discoverTerraform(root *inventory.Asset, conn *connection.GitL }}, }) } + + if discoveredTypes.k8s && slices.Contains(targets, DiscoveryK8sManifests) { + res = append(res, &inventory.Asset{ + Connections: []*inventory.Config{{ + Type: "k8s", + Options: map[string]string{ + "ssh-url": project.SSHURLToRepo, + "http-url": project.HTTPURLToRepo, + }, + Credentials: creds, + Discover: &inventory.Discovery{Targets: []string{"auto"}}, + }}, + }) + } } return res, nil } -// discoverTerraformHcl will check if the repository contains terraform files and return the terraform asset -func discoverTerraformHcl(client *gitlab.Client, pid interface{}) ([]string, error) { +type discoveredTypes struct { + terraform bool + k8s bool +} + +// discoverRepoTypes will check if the repository contains terraform files and yaml files +func discoverRepoTypes(client *gitlab.Client, pid interface{}) (*discoveredTypes, error) { opts := &gitlab.ListTreeOptions{ ListOptions: gitlab.ListOptions{ PerPage: 100, }, - Recursive: gitlab.Bool(true), + Recursive: gitlab.Ptr(true), } nodes := []*gitlab.TreeNode{} @@ -343,12 +369,34 @@ func discoverTerraformHcl(client *gitlab.Client, pid interface{}) ([]string, err } terraformFiles := []string{} + yamlFiles := []string{} for i := range nodes { node := nodes[i] + fragments := strings.Split(node.Path, "/") + isHidden := false + for _, f := range fragments { + if strings.HasPrefix(f, ".") { + isHidden = true + break + } + } + + // Skip hidden and files in hidden folders + if isHidden { + continue + } + if node.Type == "blob" && strings.HasSuffix(node.Path, ".tf") { terraformFiles = append(terraformFiles, node.Path) + } else if node.Type == "blob" && + !strings.HasSuffix(node.Path, "mql.yaml") && !strings.HasSuffix(node.Path, "mql.yml") && + (strings.HasSuffix(node.Path, ".yaml") || strings.HasSuffix(node.Path, ".yml")) { + yamlFiles = append(yamlFiles, node.Path) } } - return terraformFiles, nil + return &discoveredTypes{ + terraform: len(terraformFiles) > 0, + k8s: len(yamlFiles) > 0, + }, nil } diff --git a/providers/gitlab/provider/provider.go b/providers/gitlab/provider/provider.go index c66097e463..c16159ac20 100644 --- a/providers/gitlab/provider/provider.go +++ b/providers/gitlab/provider/provider.go @@ -30,7 +30,8 @@ const ( DiscoveryGroup = "groups" DiscoveryProject = "projects" // -- chained git discovery options -- - DiscoveryTerraform = "terraform" + DiscoveryTerraform = "terraform" + DiscoveryK8sManifests = "k8s-manifests" ) type Service struct {