From ad1c9d38e273b78276b6cb85d862463c9c72d1e5 Mon Sep 17 00:00:00 2001 From: vjeffrey Date: Tue, 26 Sep 2023 17:31:45 -0600 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B9=20update=20gitlab=20dep=20to=20lat?= =?UTF-8?q?est=20and=20clean=20up=20the=20code=20(#1941)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 4 +- go.sum | 4 ++ motor/discovery/gitlab/gitlab.go | 22 +++++++--- motor/providers/gitlab/platform.go | 30 ++++++++++++- motor/providers/gitlab/provider.go | 2 + resources/packs/gitlab/gitlab.go | 68 ++++++++++++++++++++++-------- 6 files changed, 101 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index f2aa8a0169..d07597ea20 100644 --- a/go.mod +++ b/go.mod @@ -172,7 +172,7 @@ require ( github.com/vmware/go-vcloud-director/v2 v2.17.0 github.com/vmware/goipmi v0.0.0-20181114221114-2333cd82d702 github.com/vmware/govmomi v0.29.0 - github.com/xanzy/go-gitlab v0.74.0 + github.com/xanzy/go-gitlab v0.92.1 github.com/zclconf/go-cty v1.10.0 go.mondoo.com/ranger-rpc v0.0.0-20230328135530-12135c17095f go.opentelemetry.io/otel v1.14.0 @@ -360,7 +360,7 @@ require ( github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-retryablehttp v0.7.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-secure-stdlib/mlock v0.1.1 // indirect github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect diff --git a/go.sum b/go.sum index 29c19253ab..537f8776c7 100644 --- a/go.sum +++ b/go.sum @@ -823,6 +823,8 @@ github.com/hashicorp/go-plugin v1.4.8/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHG github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-secure-stdlib/mlock v0.1.1 h1:cCRo8gK7oq6A2L6LICkUZ+/a5rLiRXFMf1Qd4xSwxTc= @@ -1424,6 +1426,8 @@ github.com/vmware/govmomi v0.29.0 h1:SHJQ7DUc4fltFZv16znJNGHR1/XhiDK5iKxm2OqwkuU github.com/vmware/govmomi v0.29.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= github.com/xanzy/go-gitlab v0.74.0 h1:Ha1cokbjn0PXy6B19t3W324dwM4AOT52fuHr7nERPrc= github.com/xanzy/go-gitlab v0.74.0/go.mod h1:d/a0vswScO7Agg1CZNz15Ic6SSvBG9vfw8egL99t4kA= +github.com/xanzy/go-gitlab v0.92.1 h1:4HfRQtGtGd1M/Xn3G6hOikfWaysL7/G6y4EEzVKINPs= +github.com/xanzy/go-gitlab v0.92.1/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/motor/discovery/gitlab/gitlab.go b/motor/discovery/gitlab/gitlab.go index 4a9dda03c8..8472354040 100644 --- a/motor/discovery/gitlab/gitlab.go +++ b/motor/discovery/gitlab/gitlab.go @@ -4,6 +4,7 @@ import ( "context" "errors" "os" + "strconv" "strings" "github.com/rs/zerolog/log" @@ -66,8 +67,10 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide case "gitlab-project": if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProject) { name := defaultName - project, _ := p.Project() - grp, _ := p.Group() + project, err := p.Project() + if err != nil { + return nil, err + } if name == "" { if project != nil { @@ -89,7 +92,7 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide list = append(list, projectAsset) if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryTerraform) { - terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), grp.Path, project.Path) + terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), project.ID) if err != nil { log.Error().Err(err).Msg("error discovering terraform") } else if len(terraformFiles) > 0 { @@ -131,13 +134,18 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide }) if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryProject) { - for _, project := range grp.Projects { + proj, err := p.GroupProjects() + if err != nil { + return nil, err + } + for _, project := range proj { clonedConfig := pCfg.Clone() if clonedConfig.Options == nil { clonedConfig.Options = map[string]string{} } clonedConfig.Options["group"] = grp.Path clonedConfig.Options["project"] = project.Path + clonedConfig.Options["project-id"] = strconv.Itoa(project.ID) id := gitlab_provider.NewGitLabProjectIdentifier(grp.Name, project.Name) projectAsset := &asset.Asset{ @@ -150,7 +158,7 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide list = append(list, projectAsset) if pCfg.IncludesOneOfDiscoveryTarget(common.DiscoveryAuto, common.DiscoveryAll, DiscoveryTerraform) { - terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), grp.Path, project.Path) + terraformFiles, err := discoverTerraformHcl(ctx, p.Client(), project.ID) if err == nil && len(terraformFiles) > 0 { terraformCfg := terraformConfig(pCfg, project.HTTPURLToRepo) terraformCfg.Credentials = credentials(pCfg) @@ -171,7 +179,7 @@ func (r *Resolver) Resolve(ctx context.Context, root *asset.Asset, pCfg *provide } // discoverTerraformHcl will check if the repository contains terraform files and return the terraform asset -func discoverTerraformHcl(ctx context.Context, client *gitlab_lib.Client, group string, project string) ([]string, error) { +func discoverTerraformHcl(ctx context.Context, client *gitlab_lib.Client, projectId int) ([]string, error) { opts := &gitlab_lib.ListTreeOptions{ ListOptions: gitlab_lib.ListOptions{ PerPage: 100, @@ -181,7 +189,7 @@ func discoverTerraformHcl(ctx context.Context, client *gitlab_lib.Client, group nodes := []*gitlab_lib.TreeNode{} for { - data, resp, err := client.Repositories.ListTree(group+"/"+project, opts) + data, resp, err := client.Repositories.ListTree(projectId, opts) if err != nil { return nil, err } diff --git a/motor/providers/gitlab/platform.go b/motor/providers/gitlab/platform.go index 10c819c960..40abbbc5de 100644 --- a/motor/providers/gitlab/platform.go +++ b/motor/providers/gitlab/platform.go @@ -53,7 +53,26 @@ func (t *Provider) Identifier() (string, error) { } func (t *Provider) Group() (*gitlab.Group, error) { - grp, _, err := t.Client().Groups.GetGroup(t.GroupPath, nil) + var gid interface{} + gid = t.GroupPath + if t.GroupId != 0 { + gid = strconv.Itoa(t.GroupId) + } + grp, _, err := t.Client().Groups.GetGroup(gid, nil) + if err != nil { + return nil, err + } + t.GroupId = grp.ID + return grp, err +} + +func (t *Provider) GroupProjects() ([]*gitlab.Project, error) { + var gid interface{} + gid = t.GroupPath + if t.GroupId != 0 { + gid = t.GroupId + } + grp, _, err := t.Client().Groups.ListGroupProjects(gid, nil) if err != nil { return nil, err } @@ -61,10 +80,17 @@ func (t *Provider) Group() (*gitlab.Group, error) { } func (t *Provider) Project() (*gitlab.Project, error) { - project, _, err := t.Client().Projects.GetProject(url.QueryEscape(t.GroupPath)+"/"+url.QueryEscape(t.ProjectPath), nil) + var pid interface{} + pid = url.QueryEscape(t.GroupPath) + "/" + url.QueryEscape(t.ProjectPath) + if t.ProjectId != 0 { + pid = t.ProjectId + } + + project, _, err := t.Client().Projects.GetProject(pid, nil) if err != nil { return nil, err } + t.ProjectId = project.ID return project, err } diff --git a/motor/providers/gitlab/provider.go b/motor/providers/gitlab/provider.go index 45b66bc6b0..220bcb5868 100644 --- a/motor/providers/gitlab/provider.go +++ b/motor/providers/gitlab/provider.go @@ -62,6 +62,8 @@ type Provider struct { opts map[string]string GroupPath string ProjectPath string + GroupId int + ProjectId int } func (p *Provider) Close() {} diff --git a/resources/packs/gitlab/gitlab.go b/resources/packs/gitlab/gitlab.go index f863cec2d9..152731b750 100644 --- a/resources/packs/gitlab/gitlab.go +++ b/resources/packs/gitlab/gitlab.go @@ -5,7 +5,6 @@ import ( "strconv" "github.com/rs/zerolog/log" - "github.com/xanzy/go-gitlab" "go.mondoo.com/cnquery/motor/providers" provider "go.mondoo.com/cnquery/motor/providers/gitlab" "go.mondoo.com/cnquery/resources" @@ -43,7 +42,7 @@ func (g *mqlGitlabGroup) init(args *resources.Args) (*resources.Args, GitlabGrou return nil, nil, err } - grp, _, err := gt.Client().Groups.GetGroup(gt.GroupPath, nil) + grp, err := gt.Group() if err != nil { return nil, nil, err } @@ -55,11 +54,7 @@ func (g *mqlGitlabGroup) init(args *resources.Args) (*resources.Args, GitlabGrou "visibility", string(grp.Visibility), "requireTwoFactorAuthentication", grp.RequireTwoFactorAuth, } - projects, err := g.createProjectResources(grp) - if err != nil { - return nil, nil, err - } - resArgs = append(resArgs, "projects", projects) + mqlGroup, err := g.MotorRuntime.CreateResource("gitlab.group", resArgs...) if err != nil { return nil, nil, err @@ -67,10 +62,15 @@ func (g *mqlGitlabGroup) init(args *resources.Args) (*resources.Args, GitlabGrou return args, mqlGroup.(*mqlGitlabGroup), nil } -func (g *mqlGitlabGroup) createProjectResources(grp *gitlab.Group) ([]interface{}, error) { +func (g *mqlGitlabGroup) createProjectResources(prov *provider.Provider) ([]interface{}, error) { var mqlProjects []interface{} - for i := range grp.Projects { - prj := grp.Projects[i] + proj, err := prov.GroupProjects() + if err != nil { + return nil, err + } + + for i := range proj { + prj := proj[i] mqlProject, err := g.MotorRuntime.CreateResource("gitlab.project", "id", int64(prj.ID), @@ -94,7 +94,11 @@ func (g *mqlGitlabGroup) createProjectResources(grp *gitlab.Group) ([]interface{ // GetProjects list all projects that belong to a group // see https://docs.gitlab.com/ee/api/projects.html func (g *mqlGitlabGroup) GetProjects() ([]interface{}, error) { - return g.Projects() + gt, err := gitlabProvider(g.MotorRuntime.Motor.Provider) + if err != nil { + return nil, err + } + return g.createProjectResources(gt) } func (g *mqlGitlabProject) id() (string, error) { @@ -107,11 +111,21 @@ func (g *mqlGitlabProject) init(args *resources.Args) (*resources.Args, GitlabPr if len(*args) > 2 { return args, nil, nil } + var projectId string + if args == nil || len(*args) == 0 { + if id := getAssetIdentifier(g.MqlResource().MotorRuntime, "project"); id != nil { + projectId = *id + } + } else { + if idArg, ok := (*args)["id"]; ok { + projectId = idArg.(string) + } + } - gt, err := gitlabProvider(g.MotorRuntime.Motor.Provider) - if err != nil { - return nil, nil, err + if projectId == "" { + return nil, nil, errors.New("no project info provided") } + obj, err := g.MotorRuntime.CreateResource("gitlab.group") if err != nil { return nil, nil, err @@ -122,18 +136,36 @@ func (g *mqlGitlabProject) init(args *resources.Args) (*resources.Args, GitlabPr if err != nil { return nil, nil, err } - matcher := gt.ProjectPath + matcher, err := strconv.Atoi(projectId) + if err != nil { + return nil, nil, err + } for i := range rawResources { proj := rawResources[i].(*mqlGitlabProject) - mqlPath, err := proj.Path() + mqlId, err := proj.Id() if err != nil { log.Error().Err(err).Msg("project is not initialized") - return nil, nil, err + continue } - if mqlPath == matcher { + if mqlId == int64(matcher) { return args, proj, nil } } return nil, nil, errors.New("project not found") } + +func getAssetIdentifier(runtime *resources.Runtime, t string) *string { + a := runtime.Motor.GetAsset() + if a == nil || len(a.Connections) == 0 { + return nil + } + switch t { + case "project": + if id, ok := a.Connections[0].Options["project-id"]; ok { + return &id + } + } + + return nil +}