⭐️ gitlab k8s manifest discovery (#4146)

Signed-off-by: Ivan Milchev <[email protected]>

explorer/scan/discovery.go

@@ -151,6 +151,12 @@ func DiscoverAssets(ctx context.Context, inv *inventory.Inventory, upstream *ups
 }
 
 func discoverAssets(rootAssetWithRuntime *AssetWithRuntime, resolvedRootAsset *inventory.Asset, discoveredAssets *DiscoveredAssets, runtimeLabels map[string]string, upstream *upstream.UpstreamConfig, recording llx.Recording) {
+	// It is possible that we did not discover any assets under the root asset. In that case the inventory
+	// would be nil and we can return
+	if rootAssetWithRuntime.Runtime.Provider.Connection.Inventory == nil {
+		return
+	}
+
 	// for all discovered assets, we apply mondoo-specific labels and annotations that come from the root asset
 	for _, a := range rootAssetWithRuntime.Runtime.Provider.Connection.Inventory.Spec.Assets {
 		// create runtime for root asset

providers/github/resources/discovery.go

@@ -336,7 +336,26 @@ func hasTerraformHcl(client *github.Client, repo *mqlGithubRepository) (bool, er
 	if err != nil {
 		return false, err
 	}
-	return res.GetTotal() > 0, nil
+
+	// Ignore tf files that are hidden or are in a hidden folder
+	nonHiddenTf := 0
+	for _, code := range res.CodeResults {
+		fragments := strings.Split(code.GetPath(), "/")
+		// skip hidden files
+		isHidden := false
+		for _, fragment := range fragments {
+			if strings.HasPrefix(fragment, ".") {
+				isHidden = true
+				break
+			}
+		}
+
+		if !isHidden {
+			nonHiddenTf++
+		}
+	}
+
+	return nonHiddenTf > 0, nil
 }
 
 func discoverK8sManifests(conn *connection.GithubConnection, repo *mqlGithubRepository) ([]*inventory.Asset, error) {
@@ -383,6 +402,13 @@ func hasYaml(client *github.Client, repo *mqlGithubRepository) (bool, error) {
 	// Ignore YAML files that are hidden or are in a hidden folder
 	nonHiddenYaml := 0
 	for _, code := range res.CodeResults {
+		path := code.GetPath()
+
+		// Skip MQL files
+		if strings.HasSuffix(path, "mql.yaml") || strings.HasSuffix(path, "mql.yml") {
+			continue
+		}
+
 		fragments := strings.Split(code.GetPath(), "/")
 		// skip hidden files
 		isHidden := false

providers/gitlab/provider/discovery.go

@@ -77,8 +77,8 @@ func (s *Service) discover(root *inventory.Asset, conn *connection.GitLabConnect
 		}
 	}
 
-	if slices.Contains(targets, DiscoveryTerraform) {
-		repos, err := s.discoverTerraform(root, conn, projects)
+	if slices.Contains(targets, DiscoveryTerraform) || slices.Contains(targets, DiscoveryK8sManifests) {
+		repos, err := s.discoverTypes(targets, conn, projects)
 		if err != nil {
 			return nil, err
 		}
@@ -279,7 +279,11 @@ func listAllGroups(conn *connection.GitLabConnection) ([]*gitlab.Group, error) {
 	return groups, nil
 }
 
-func (s *Service) discoverTerraform(root *inventory.Asset, conn *connection.GitLabConnection, projects []*gitlab.Project) ([]*inventory.Asset, error) {
+func (s *Service) discoverTypes(targets []string, conn *connection.GitLabConnection, projects []*gitlab.Project) ([]*inventory.Asset, error) {
+	if !slices.Contains(targets, DiscoveryTerraform) && !slices.Contains(targets, DiscoveryK8sManifests) {
+		return nil, nil
+	}
+
 	// For git clone we need to set the user to oauth2 to be usable with the token.
 	creds := make([]*vault.Credential, len(conn.Conf.Credentials))
 	for i := range conn.Conf.Credentials {
@@ -294,10 +298,13 @@ func (s *Service) discoverTerraform(root *inventory.Asset, conn *connection.GitL
 	var res []*inventory.Asset
 	for i := range projects {
 		project := projects[i]
-		files, err := discoverTerraformHcl(conn.Client(), project.ID)
+		discoveredTypes, err := discoverRepoTypes(conn.Client(), project.ID)
 		if err != nil {
 			log.Error().Err(err).Str("project", project.PathWithNamespace).Msg("failed to discover terraform repo in gitlab")
-		} else if len(files) != 0 {
+			continue
+		}
+
+		if discoveredTypes.terraform && slices.Contains(targets, DiscoveryTerraform) {
 			res = append(res, &inventory.Asset{
 				Connections: []*inventory.Config{{
 					Type: "terraform-hcl-git",
@@ -309,17 +316,36 @@ func (s *Service) discoverTerraform(root *inventory.Asset, conn *connection.GitL
 				}},
 			})
 		}
+
+		if discoveredTypes.k8s && slices.Contains(targets, DiscoveryK8sManifests) {
+			res = append(res, &inventory.Asset{
+				Connections: []*inventory.Config{{
+					Type: "k8s",
+					Options: map[string]string{
+						"ssh-url":  project.SSHURLToRepo,
+						"http-url": project.HTTPURLToRepo,
+					},
+					Credentials: creds,
+					Discover:    &inventory.Discovery{Targets: []string{"auto"}},
+				}},
+			})
+		}
 	}
 	return res, nil
 }
 
-// discoverTerraformHcl will check if the repository contains terraform files and return the terraform asset
-func discoverTerraformHcl(client *gitlab.Client, pid interface{}) ([]string, error) {
+type discoveredTypes struct {
+	terraform bool
+	k8s       bool
+}
+
+// discoverRepoTypes will check if the repository contains terraform files and yaml files
+func discoverRepoTypes(client *gitlab.Client, pid interface{}) (*discoveredTypes, error) {
 	opts := &gitlab.ListTreeOptions{
 		ListOptions: gitlab.ListOptions{
 			PerPage: 100,
 		},
-		Recursive: gitlab.Bool(true),
+		Recursive: gitlab.Ptr(true),
 	}
 
 	nodes := []*gitlab.TreeNode{}
@@ -343,12 +369,34 @@ func discoverTerraformHcl(client *gitlab.Client, pid interface{}) ([]string, err
 	}
 
 	terraformFiles := []string{}
+	yamlFiles := []string{}
 	for i := range nodes {
 		node := nodes[i]
+		fragments := strings.Split(node.Path, "/")
+		isHidden := false
+		for _, f := range fragments {
+			if strings.HasPrefix(f, ".") {
+				isHidden = true
+				break
+			}
+		}
+
+		// Skip hidden and files in hidden folders
+		if isHidden {
+			continue
+		}
+
 		if node.Type == "blob" && strings.HasSuffix(node.Path, ".tf") {
 			terraformFiles = append(terraformFiles, node.Path)
+		} else if node.Type == "blob" &&
+			!strings.HasSuffix(node.Path, "mql.yaml") && !strings.HasSuffix(node.Path, "mql.yml") &&
+			(strings.HasSuffix(node.Path, ".yaml") || strings.HasSuffix(node.Path, ".yml")) {
+			yamlFiles = append(yamlFiles, node.Path)
 		}
 	}
 
-	return terraformFiles, nil
+	return &discoveredTypes{
+		terraform: len(terraformFiles) > 0,
+		k8s:       len(yamlFiles) > 0,
+	}, nil
 }

providers/gitlab/provider/provider.go

@@ -30,7 +30,8 @@ const (
 	DiscoveryGroup   = "groups"
 	DiscoveryProject = "projects"
 	// -- chained git discovery options --
-	DiscoveryTerraform = "terraform"
+	DiscoveryTerraform    = "terraform"
+	DiscoveryK8sManifests = "k8s-manifests"
 )
 
 type Service struct {