From d13568792cd8887564e936a7fe1a44aa22890623 Mon Sep 17 00:00:00 2001 From: Jay Mundrawala Date: Mon, 23 Sep 2024 12:16:32 -0500 Subject: [PATCH] =?UTF-8?q?=E2=AD=90=EF=B8=8F=20Optimized=20fields=20for?= =?UTF-8?q?=20github=20provider=20(#4672)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add fields to fetch suport md file and code of conduct It should be more efficient to fetch a single file than to try to list all the files and then try to have cnquery filter for it. * Add field for admin collaborators This should list fewer collaborators than trying to list all of them. Useful for queries that only care about admins * Add field for security.md file * Tim's comments about adding File suffix * Update providers/github/resources/github.lr Co-authored-by: Letha --------- Co-authored-by: Tim Smith Co-authored-by: Letha --- providers/github/resources/github.lr | 12 +- providers/github/resources/github.lr.go | 110 +++++++++- .../github/resources/github.lr.manifest.yaml | 31 +++ providers/github/resources/github_repo.go | 205 +++++++++++++++++- 4 files changed, 355 insertions(+), 3 deletions(-) diff --git a/providers/github/resources/github.lr b/providers/github/resources/github.lr index 208c545025..300fbfe5da 100644 --- a/providers/github/resources/github.lr +++ b/providers/github/resources/github.lr @@ -322,8 +322,10 @@ private github.repository @defaults("fullName") { commits() []github.commit // List of contributors for the repository contributors() []github.user - // List of collaborators for the repository + // List of all collaborators for the repository collaborators() []github.collaborator + // List of admin collaborators for the repository + adminCollaborators() []github.collaborator // List of files in the repository files() []github.file // List of releases for the repository @@ -344,6 +346,12 @@ private github.repository @defaults("fullName") { closedIssues() []github.issue // Repository license license() github.license + // Repository code of conduct + codeOfConductFile() github.file + // Repository support file + supportFile() github.file + // Repository security file + securityFile() github.file } // GitHub license @@ -380,6 +388,8 @@ private github.file @defaults("name type") { content() string // File download URL downloadUrl string + // Whether the file exists in the repository + exists bool } // GitHub release diff --git a/providers/github/resources/github.lr.go b/providers/github/resources/github.lr.go index aa0cca7ccb..beb8b4516f 100644 --- a/providers/github/resources/github.lr.go +++ b/providers/github/resources/github.lr.go @@ -606,6 +606,9 @@ var getDataFields = map[string]func(r plugin.Resource) *plugin.DataRes{ "github.repository.collaborators": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlGithubRepository).GetCollaborators()).ToDataRes(types.Array(types.Resource("github.collaborator"))) }, + "github.repository.adminCollaborators": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlGithubRepository).GetAdminCollaborators()).ToDataRes(types.Array(types.Resource("github.collaborator"))) + }, "github.repository.files": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlGithubRepository).GetFiles()).ToDataRes(types.Array(types.Resource("github.file"))) }, @@ -636,6 +639,15 @@ var getDataFields = map[string]func(r plugin.Resource) *plugin.DataRes{ "github.repository.license": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlGithubRepository).GetLicense()).ToDataRes(types.Resource("github.license")) }, + "github.repository.codeOfConductFile": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlGithubRepository).GetCodeOfConductFile()).ToDataRes(types.Resource("github.file")) + }, + "github.repository.supportFile": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlGithubRepository).GetSupportFile()).ToDataRes(types.Resource("github.file")) + }, + "github.repository.securityFile": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlGithubRepository).GetSecurityFile()).ToDataRes(types.Resource("github.file")) + }, "github.license.key": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlGithubLicense).GetKey()).ToDataRes(types.String) }, @@ -678,6 +690,9 @@ var getDataFields = map[string]func(r plugin.Resource) *plugin.DataRes{ "github.file.downloadUrl": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlGithubFile).GetDownloadUrl()).ToDataRes(types.String) }, + "github.file.exists": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlGithubFile).GetExists()).ToDataRes(types.Bool) + }, "github.release.url": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlGithubRelease).GetUrl()).ToDataRes(types.String) }, @@ -1567,6 +1582,10 @@ var setDataFields = map[string]func(r plugin.Resource, v *llx.RawData) bool { r.(*mqlGithubRepository).Collaborators, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) return }, + "github.repository.adminCollaborators": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlGithubRepository).AdminCollaborators, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) + return + }, "github.repository.files": func(r plugin.Resource, v *llx.RawData) (ok bool) { r.(*mqlGithubRepository).Files, ok = plugin.RawToTValue[[]interface{}](v.Value, v.Error) return @@ -1607,6 +1626,18 @@ var setDataFields = map[string]func(r plugin.Resource, v *llx.RawData) bool { r.(*mqlGithubRepository).License, ok = plugin.RawToTValue[*mqlGithubLicense](v.Value, v.Error) return }, + "github.repository.codeOfConductFile": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlGithubRepository).CodeOfConductFile, ok = plugin.RawToTValue[*mqlGithubFile](v.Value, v.Error) + return + }, + "github.repository.supportFile": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlGithubRepository).SupportFile, ok = plugin.RawToTValue[*mqlGithubFile](v.Value, v.Error) + return + }, + "github.repository.securityFile": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlGithubRepository).SecurityFile, ok = plugin.RawToTValue[*mqlGithubFile](v.Value, v.Error) + return + }, "github.license.__id": func(r plugin.Resource, v *llx.RawData) (ok bool) { r.(*mqlGithubLicense).__id, ok = v.Value.(string) return @@ -1671,6 +1702,10 @@ var setDataFields = map[string]func(r plugin.Resource, v *llx.RawData) bool { r.(*mqlGithubFile).DownloadUrl, ok = plugin.RawToTValue[string](v.Value, v.Error) return }, + "github.file.exists": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlGithubFile).Exists, ok = plugin.RawToTValue[bool](v.Value, v.Error) + return + }, "github.release.__id": func(r plugin.Resource, v *llx.RawData) (ok bool) { r.(*mqlGithubRelease).__id, ok = v.Value.(string) return @@ -3228,7 +3263,7 @@ func (c *mqlGithubPackages) GetList() *plugin.TValue[[]interface{}] { type mqlGithubRepository struct { MqlRuntime *plugin.Runtime __id string - // optional: if you define mqlGithubRepositoryInternal it will be used here + mqlGithubRepositoryInternal Id plugin.TValue[int64] Name plugin.TValue[string] FullName plugin.TValue[string] @@ -3271,6 +3306,7 @@ type mqlGithubRepository struct { Commits plugin.TValue[[]interface{}] Contributors plugin.TValue[[]interface{}] Collaborators plugin.TValue[[]interface{}] + AdminCollaborators plugin.TValue[[]interface{}] Files plugin.TValue[[]interface{}] Releases plugin.TValue[[]interface{}] Owner plugin.TValue[*mqlGithubUser] @@ -3281,6 +3317,9 @@ type mqlGithubRepository struct { OpenIssues plugin.TValue[[]interface{}] ClosedIssues plugin.TValue[[]interface{}] License plugin.TValue[*mqlGithubLicense] + CodeOfConductFile plugin.TValue[*mqlGithubFile] + SupportFile plugin.TValue[*mqlGithubFile] + SecurityFile plugin.TValue[*mqlGithubFile] } // createGithubRepository creates a new instance of this resource @@ -3584,6 +3623,22 @@ func (c *mqlGithubRepository) GetCollaborators() *plugin.TValue[[]interface{}] { }) } +func (c *mqlGithubRepository) GetAdminCollaborators() *plugin.TValue[[]interface{}] { + return plugin.GetOrCompute[[]interface{}](&c.AdminCollaborators, func() ([]interface{}, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("github.repository", c.__id, "adminCollaborators") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.([]interface{}), nil + } + } + + return c.adminCollaborators() + }) +} + func (c *mqlGithubRepository) GetFiles() *plugin.TValue[[]interface{}] { return plugin.GetOrCompute[[]interface{}](&c.Files, func() ([]interface{}, error) { if c.MqlRuntime.HasRecording { @@ -3732,6 +3787,54 @@ func (c *mqlGithubRepository) GetLicense() *plugin.TValue[*mqlGithubLicense] { }) } +func (c *mqlGithubRepository) GetCodeOfConductFile() *plugin.TValue[*mqlGithubFile] { + return plugin.GetOrCompute[*mqlGithubFile](&c.CodeOfConductFile, func() (*mqlGithubFile, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("github.repository", c.__id, "codeOfConductFile") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.(*mqlGithubFile), nil + } + } + + return c.codeOfConductFile() + }) +} + +func (c *mqlGithubRepository) GetSupportFile() *plugin.TValue[*mqlGithubFile] { + return plugin.GetOrCompute[*mqlGithubFile](&c.SupportFile, func() (*mqlGithubFile, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("github.repository", c.__id, "supportFile") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.(*mqlGithubFile), nil + } + } + + return c.supportFile() + }) +} + +func (c *mqlGithubRepository) GetSecurityFile() *plugin.TValue[*mqlGithubFile] { + return plugin.GetOrCompute[*mqlGithubFile](&c.SecurityFile, func() (*mqlGithubFile, error) { + if c.MqlRuntime.HasRecording { + d, err := c.MqlRuntime.FieldResourceFromRecording("github.repository", c.__id, "securityFile") + if err != nil { + return nil, err + } + if d != nil { + return d.Value.(*mqlGithubFile), nil + } + } + + return c.securityFile() + }) +} + // mqlGithubLicense for the github.license resource type mqlGithubLicense struct { MqlRuntime *plugin.Runtime @@ -3811,6 +3914,7 @@ type mqlGithubFile struct { RepoName plugin.TValue[string] Content plugin.TValue[string] DownloadUrl plugin.TValue[string] + Exists plugin.TValue[bool] } // createGithubFile creates a new instance of this resource @@ -3904,6 +4008,10 @@ func (c *mqlGithubFile) GetDownloadUrl() *plugin.TValue[string] { return &c.DownloadUrl } +func (c *mqlGithubFile) GetExists() *plugin.TValue[bool] { + return &c.Exists +} + // mqlGithubRelease for the github.release resource type mqlGithubRelease struct { MqlRuntime *plugin.Runtime diff --git a/providers/github/resources/github.lr.manifest.yaml b/providers/github/resources/github.lr.manifest.yaml index 2a03b4cc7a..a5f4adf8dd 100755 --- a/providers/github/resources/github.lr.manifest.yaml +++ b/providers/github/resources/github.lr.manifest.yaml @@ -105,6 +105,8 @@ resources: content: {} downloadUrl: min_mondoo_version: 9.0.0 + exists: + min_mondoo_version: 9.0.0 files: {} isBinary: {} name: @@ -316,6 +318,8 @@ resources: fields: MergeRequests: min_mondoo_version: 9.0.0 + adminCollaborators: + min_mondoo_version: 9.0.0 allMergeRequests: min_mondoo_version: 9.0.0 allowAutoMerge: @@ -337,6 +341,10 @@ resources: min_mondoo_version: 7.14.0 closedMergeRequests: min_mondoo_version: 9.0.0 + codeOfConduct: + min_mondoo_version: 9.0.0 + codeOfConductFile: + min_mondoo_version: 9.0.0 collaborators: min_mondoo_version: 6.11.0 commits: @@ -393,14 +401,24 @@ resources: private: {} pushedAt: min_mondoo_version: 7.14.0 + readme: + min_mondoo_version: 9.0.0 releases: min_mondoo_version: 6.4.0 + security: + min_mondoo_version: 9.0.0 + securityFile: + min_mondoo_version: 9.0.0 sshUrl: min_mondoo_version: 7.14.0 stargazers: min_mondoo_version: 7.14.0 stargazersCount: min_mondoo_version: 7.14.0 + support: + min_mondoo_version: 9.0.0 + supportFile: + min_mondoo_version: 9.0.0 topics: min_mondoo_version: 7.14.0 updatedAt: {} @@ -413,6 +431,19 @@ resources: min_mondoo_version: 6.11.0 is_private: true min_mondoo_version: 5.31.0 + github.repository.file: + fields: + content: {} + downloadUrl: {} + files: {} + isBinary: {} + name: {} + ownerName: {} + path: {} + repoName: {} + sha: {} + type: {} + min_mondoo_version: 9.0.0 github.review: fields: authorAssociation: {} diff --git a/providers/github/resources/github_repo.go b/providers/github/resources/github_repo.go index 34bebfaf8f..47b9e6cdc9 100644 --- a/providers/github/resources/github_repo.go +++ b/providers/github/resources/github_repo.go @@ -6,8 +6,10 @@ package resources import ( "errors" "fmt" + "path" "strconv" "strings" + "sync" "github.com/google/go-github/v62/github" "github.com/rs/zerolog/log" @@ -72,7 +74,9 @@ func newMqlGithubRepository(runtime *plugin.Runtime, repo *github.Repository) (* if err != nil { return nil, err } - return res.(*mqlGithubRepository), nil + r := res.(*mqlGithubRepository) + r.mqlGithubRepositoryInternal.findSpecialFilesOnceFunc = sync.OnceValue[error](r.findSpecialFiles) + return r, nil } func (g *mqlGithubBranchprotection) id() (string, error) { @@ -972,6 +976,70 @@ func (g *mqlGithubRepository) collaborators() ([]interface{}, error) { return res, nil } +func (g *mqlGithubRepository) adminCollaborators() ([]interface{}, error) { + conn := g.MqlRuntime.Connection.(*connection.GithubConnection) + + if g.Name.Error != nil { + return nil, g.Name.Error + } + repoName := g.Name.Data + if g.Owner.Error != nil { + return nil, g.Owner.Error + } + ownerName := g.Owner.Data + if ownerName.Login.Error != nil { + return nil, ownerName.Login.Error + } + ownerLogin := ownerName.Login.Data + + listOpts := &github.ListCollaboratorsOptions{ + Permission: "admin", + ListOptions: github.ListOptions{PerPage: paginationPerPage}, + } + var adminCollaborators []*github.User + for { + contributors, resp, err := conn.Client().Repositories.ListCollaborators(conn.Context(), ownerLogin, repoName, listOpts) + if err != nil { + if strings.Contains(err.Error(), "404") { + return nil, nil + } + return nil, err + } + adminCollaborators = append(adminCollaborators, contributors...) + if resp.NextPage == 0 { + break + } + listOpts.Page = resp.NextPage + } + res := []interface{}{} + for i := range adminCollaborators { + contributor := adminCollaborators[i] + mqlUser, err := NewResource(g.MqlRuntime, "github.user", map[string]*llx.RawData{ + "id": llx.IntDataPtr(contributor.ID), + "login": llx.StringDataPtr(contributor.Login), + }) + if err != nil { + return nil, err + } + + permissions := []string{} + for k := range contributor.Permissions { + permissions = append(permissions, k) + } + + mqlContributor, err := CreateResource(g.MqlRuntime, "github.collaborator", map[string]*llx.RawData{ + "id": llx.IntDataPtr(contributor.ID), + "user": llx.ResourceData(mqlUser, mqlUser.MqlName()), + "permissions": llx.ArrayData(convert.SliceAnyToInterface[string](permissions), types.String), + }) + if err != nil { + return nil, err + } + res = append(res, mqlContributor) + } + return res, nil +} + func (g *mqlGithubRepository) releases() ([]interface{}, error) { conn := g.MqlRuntime.Connection.(*connection.GithubConnection) @@ -1177,6 +1245,27 @@ func newMqlGithubFile(runtime *plugin.Runtime, ownerName string, repoName string "ownerName": llx.StringData(ownerName), "repoName": llx.StringData(repoName), "downloadUrl": llx.StringDataPtr(content.DownloadURL), + "exists": llx.BoolData(true), + }) + if err != nil { + return nil, err + } + return res.(*mqlGithubFile), nil +} + +func newMqlGithubFileDoesNotExist(runtime *plugin.Runtime, ownerName string, repoName string, filePath string) (*mqlGithubFile, error) { + name := path.Base(filePath) + + res, err := CreateResource(runtime, "github.file", map[string]*llx.RawData{ + "path": llx.StringData(filePath), + "name": llx.StringData(name), + "type": llx.StringDataPtr(nil), + "sha": llx.StringData("0000000000000000000000000000000000000000"), + "isBinary": llx.BoolData(false), + "ownerName": llx.StringData(ownerName), + "repoName": llx.StringData(repoName), + "downloadUrl": llx.StringDataPtr(nil), + "exists": llx.BoolData(false), }) if err != nil { return nil, err @@ -1568,3 +1657,117 @@ func (g *mqlGithubRepository) getIssues(state string) ([]interface{}, error) { } return res, nil } + +func (g *mqlGithubRepository) supportFile() (*mqlGithubFile, error) { + err := g.findSpecialFilesOnceFunc() + if err != nil { + return nil, err + } + if g.SupportFile.Error != nil { + return nil, g.SupportFile.Error + } + + return g.SupportFile.Data, nil +} + +func (g *mqlGithubRepository) codeOfConductFile() (*mqlGithubFile, error) { + err := g.findSpecialFilesOnceFunc() + if err != nil { + return nil, err + } + if g.CodeOfConductFile.Error != nil { + return nil, g.CodeOfConductFile.Error + } + + return g.CodeOfConductFile.Data, nil +} + +func (g *mqlGithubRepository) securityFile() (*mqlGithubFile, error) { + err := g.findSpecialFilesOnceFunc() + if err != nil { + return nil, err + } + if g.SecurityFile.Error != nil { + return nil, g.SecurityFile.Error + } + + return g.SecurityFile.Data, nil +} + +type mqlGithubRepositoryInternal struct { + findSpecialFilesOnceFunc func() error +} + +func (g *mqlGithubRepository) findSpecialFiles() error { + conn := g.MqlRuntime.Connection.(*connection.GithubConnection) + + if g.Name.Error != nil { + return g.Name.Error + } + repoName := g.Name.Data + + if g.Owner.Error != nil { + return g.Owner.Error + } + owner := g.Owner.Data + if owner.Login.Error != nil { + return owner.Login.Error + } + ownerLogin := owner.Login.Data + + specialDirectories := []string{".", ".github"} + + specialFilesCaseInsensitive := map[string]*plugin.TValue[*mqlGithubFile]{ + "code_of_conduct.md": &g.CodeOfConductFile, + "support.md": &g.SupportFile, + "security.md": &g.SecurityFile, + } + foundFiles := map[string]struct{}{} + + for _, dir := range specialDirectories { + if len(foundFiles) == len(specialFilesCaseInsensitive) { + break + } + _, dirContent, _, err := conn.Client().Repositories.GetContents(conn.Context(), ownerLogin, repoName, dir, &github.RepositoryContentGetOptions{}) + if err != nil { + if strings.Contains(err.Error(), "404") { + continue + } + return err + } + + for i := range dirContent { + if dirContent[i].GetType() != "file" { + continue + } + if _, ok := foundFiles[dirContent[i].GetName()]; ok { + continue + } + + name := strings.ToLower(dirContent[i].GetName()) + if _, ok := specialFilesCaseInsensitive[name]; ok { + v := specialFilesCaseInsensitive[name] + if v != nil { + file, err := newMqlGithubFile(g.MqlRuntime, ownerLogin, repoName, dirContent[i]) + if err != nil { + return err + } + (*v) = plugin.TValue[*mqlGithubFile]{Data: file, State: plugin.StateIsSet, Error: nil} + } + foundFiles[name] = struct{}{} + } + } + } + + for k, v := range specialFilesCaseInsensitive { + if _, ok := foundFiles[k]; !ok { + dneFile, err := newMqlGithubFileDoesNotExist(g.MqlRuntime, ownerLogin, repoName, k) + if err != nil { + return err + } + (*v) = plugin.TValue[*mqlGithubFile]{Data: dneFile, State: plugin.StateIsSet, Error: nil} + } + } + + return nil +}