From 307d150cb770f7c1ec712f42f553e6a0b6ae81db Mon Sep 17 00:00:00 2001 From: Tim Smith Date: Thu, 21 Sep 2023 15:28:51 -0700 Subject: [PATCH] Update GitHub library to v55 There's a very large number of improvements here including handling of secondary API limits Signed-off-by: Tim Smith --- providers/github/connection/connection.go | 2 +- providers/github/connection/connection_test.go | 2 +- providers/github/connection/platform.go | 2 +- providers/github/go.mod | 13 +++++++++---- providers/github/go.sum | 17 +++++++++++------ providers/github/resources/git.go | 4 ++-- providers/github/resources/github.go | 2 +- providers/github/resources/github_org.go | 12 ++++++------ providers/github/resources/github_repo.go | 10 +++++----- providers/github/resources/github_team.go | 2 +- providers/github/resources/github_user.go | 6 +++--- providers/github/resources/github_workflow.go | 2 +- 12 files changed, 42 insertions(+), 32 deletions(-) diff --git a/providers/github/connection/connection.go b/providers/github/connection/connection.go index 6cc289c4fd..f7c6181e22 100644 --- a/providers/github/connection/connection.go +++ b/providers/github/connection/connection.go @@ -9,7 +9,7 @@ import ( "os" "github.com/cockroachdb/errors" - "github.com/google/go-github/v49/github" + "github.com/google/go-github/v55/github" "github.com/rs/zerolog/log" "go.mondoo.com/cnquery/providers-sdk/v1/inventory" "go.mondoo.com/cnquery/providers-sdk/v1/vault" diff --git a/providers/github/connection/connection_test.go b/providers/github/connection/connection_test.go index 1d62d2e555..82eba0a06d 100644 --- a/providers/github/connection/connection_test.go +++ b/providers/github/connection/connection_test.go @@ -11,7 +11,7 @@ import ( "os" "testing" - "github.com/google/go-github/v49/github" + "github.com/google/go-github/v55/github" "github.com/stretchr/testify/require" "go.mondoo.com/cnquery/providers-sdk/v1/inventory" ) diff --git a/providers/github/connection/platform.go b/providers/github/connection/platform.go index a29e8c9389..cc2a123890 100644 --- a/providers/github/connection/platform.go +++ b/providers/github/connection/platform.go @@ -7,7 +7,7 @@ import ( "context" "github.com/cockroachdb/errors" - "github.com/google/go-github/v49/github" + "github.com/google/go-github/v55/github" "go.mondoo.com/cnquery/providers-sdk/v1/inventory" ) diff --git a/providers/github/go.mod b/providers/github/go.mod index 7ea1c07a46..23e94f9fcb 100644 --- a/providers/github/go.mod +++ b/providers/github/go.mod @@ -6,10 +6,10 @@ go 1.20 require ( github.com/cockroachdb/errors v1.11.1 - github.com/google/go-github/v49 v49.1.0 + github.com/google/go-github/v55 v55.0.0 github.com/rs/zerolog v1.30.0 github.com/stretchr/testify v1.8.4 - go.mondoo.com/cnquery v0.0.0-20230915180754-c5f61bc705cf + go.mondoo.com/cnquery v0.0.0-20230921152023-8e36364513b6 go.mondoo.com/ranger-rpc v0.5.1 golang.org/x/oauth2 v0.12.0 sigs.k8s.io/yaml v1.3.0 @@ -86,6 +86,7 @@ require ( github.com/docker/docker-credential-helpers v0.8.0 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/esimonov/ifshort v1.0.4 // indirect github.com/ettle/strcase v0.1.1 // indirect github.com/fatih/color v1.15.0 // indirect @@ -94,6 +95,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect github.com/getsentry/sentry-go v0.24.1 // indirect + github.com/glebarez/go-sqlite v1.21.2 // indirect github.com/go-critic/go-critic v0.9.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -126,7 +128,6 @@ require ( github.com/google/go-containerregistry v0.16.1 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect github.com/google/uuid v1.3.1 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/gordonklaus/ineffassign v0.0.0-20230610083614-0e73809eb601 // indirect @@ -271,7 +272,7 @@ require ( golang.org/x/tools v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect google.golang.org/grpc v1.58.1 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -288,6 +289,10 @@ require ( k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kubelet v0.28.2 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + modernc.org/libc v1.24.1 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.7.1 // indirect + modernc.org/sqlite v1.25.0 // indirect moul.io/http2curl v1.0.0 // indirect mvdan.cc/gofumpt v0.5.0 // indirect mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect diff --git a/providers/github/go.sum b/providers/github/go.sum index c75f173c1a..2031acf9ae 100644 --- a/providers/github/go.sum +++ b/providers/github/go.sum @@ -208,6 +208,7 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -232,7 +233,8 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= github.com/getsentry/sentry-go v0.24.1 h1:W6/0GyTy8J6ge6lVCc94WB6Gx2ZuLrgopnn9w8Hiwuk= github.com/getsentry/sentry-go v0.24.1/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= -github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4= +github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= +github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= github.com/go-critic/go-critic v0.9.0 h1:Pmys9qvU3pSML/3GEQ2Xd9RZ/ip+aXHKILuxczKGV/U= github.com/go-critic/go-critic v0.9.0/go.mod h1:5P8tdXL7m/6qnyG6oRAlYLORvoXH0WDypYgAEmagT40= github.com/go-errors/errors v1.5.0 h1:/EuijeGOu7ckFxzhkj4CXJ8JaenxK7bKUxpPYqeLHqQ= @@ -354,8 +356,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ= github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ= -github.com/google/go-github/v49 v49.1.0 h1:LFkMgawGQ8dfzWLH/rNE0b3u1D3n6/dw7ZmrN3b+YFY= -github.com/google/go-github/v49 v49.1.0/go.mod h1:MUUzHPrhGniB6vUKa27y37likpipzG+BXXJbG04J334= +github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg= +github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -375,7 +377,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= @@ -1153,8 +1154,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb h1:Isk1sSH7bovx8Rti2wZK0UZF6oraBDK74uoyLEEVFN0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1239,9 +1240,13 @@ k8s.io/kubelet v0.28.2/go.mod h1:rvd0e7T5TjPcfZvy62P90XhFzp0IhPIOy+Pqy3Rtipo= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= +modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.7.1 h1:9J+2/GKTlV503mk3yv8QJ6oEpRCUrRy0ad8TXEPoV8M= +modernc.org/memory v1.7.1/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA= +modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU= moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E= diff --git a/providers/github/resources/git.go b/providers/github/resources/git.go index a47eacf31a..aa5ee28df4 100644 --- a/providers/github/resources/git.go +++ b/providers/github/resources/git.go @@ -4,7 +4,7 @@ package resources import ( - "github.com/google/go-github/v49/github" + "github.com/google/go-github/v55/github" "go.mondoo.com/cnquery/llx" "go.mondoo.com/cnquery/providers-sdk/v1/plugin" ) @@ -33,7 +33,7 @@ func newMqlGitAuthor(runtime *plugin.Runtime, sha string, a *github.CommitAuthor "sha": llx.StringData(sha), "name": llx.StringData(a.GetName()), "email": llx.StringData(a.GetEmail()), - "date": llx.TimeData(date), + "date": llx.TimeData(githubTimestamp(date)), }) } diff --git a/providers/github/resources/github.go b/providers/github/resources/github.go index 051589cf79..214c1a6b30 100644 --- a/providers/github/resources/github.go +++ b/providers/github/resources/github.go @@ -6,7 +6,7 @@ package resources import ( "time" - "github.com/google/go-github/v49/github" + "github.com/google/go-github/v55/github" ) func githubTimestamp(ts *github.Timestamp) *time.Time { diff --git a/providers/github/resources/github_org.go b/providers/github/resources/github_org.go index 3859ccf08d..ffb4dc5cc3 100644 --- a/providers/github/resources/github_org.go +++ b/providers/github/resources/github_org.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/google/go-github/v49/github" + "github.com/google/go-github/v55/github" "go.mondoo.com/cnquery/llx" "go.mondoo.com/cnquery/providers-sdk/v1/plugin" "go.mondoo.com/cnquery/providers-sdk/v1/util/convert" @@ -43,7 +43,7 @@ func initGithubOrganization(runtime *plugin.Runtime, args map[string]*llx.RawDat if name == "" { name = *org.Name } - args["id"] = llx.IntData(convert.ToInt64(org.ID)) + args["id"] = llx.IntData(org.ID) args["name"] = llx.StringData(name) args["login"] = llx.StringDataPtr(org.Login) args["nodeId"] = llx.StringDataPtr(org.NodeID) @@ -56,10 +56,10 @@ func initGithubOrganization(runtime *plugin.Runtime, args map[string]*llx.RawDat args["followers"] = llx.IntData(convert.ToInt64FromInt(org.Followers)) args["following"] = llx.IntData(convert.ToInt64FromInt(org.Following)) args["description"] = llx.StringDataPtr(org.Description) - args["createdAt"] = llx.TimeDataPtr(org.CreatedAt) - args["updatedAt"] = llx.TimeDataPtr(org.UpdatedAt) - args["totalPrivateRepos"] = llx.IntData(convert.ToInt64FromInt(org.TotalPrivateRepos)) - args["ownedPrivateRepos"] = llx.IntData(convert.ToInt64FromInt(org.OwnedPrivateRepos)) + args["createdAt"] = llx.TimeDataPtr(githubTimestamp(org.CreatedAt)) + args["updatedAt"] = llx.TimeDataPtr(githubTimestamp(org.UpdatedAt)) + args["totalPrivateRepos"] = llx.IntData(org.TotalPrivateRepos) + args["ownedPrivateRepos"] = llx.IntData(org.OwnedPrivateRepos) args["privateGists"] = llx.IntData(convert.ToInt64FromInt(org.PrivateGists)) args["diskUsage"] = llx.IntData(convert.ToInt64FromInt(org.DiskUsage)) args["collaborators"] = llx.IntData(convert.ToInt64FromInt(org.Collaborators)) diff --git a/providers/github/resources/github_repo.go b/providers/github/resources/github_repo.go index 820c28819f..333dc3a876 100644 --- a/providers/github/resources/github_repo.go +++ b/providers/github/resources/github_repo.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/google/go-github/v49/github" + "github.com/google/go-github/v55/github" "github.com/rs/zerolog/log" "go.mondoo.com/cnquery/llx" "go.mondoo.com/cnquery/providers-sdk/v1/plugin" @@ -331,7 +331,7 @@ func (g *mqlGithubRepository) openMergeRequests() ([]interface{}, error) { "number": llx.IntData(int64(*pr.Number)), "state": llx.StringDataPtr(pr.State), "labels": llx.ArrayData(labels, types.Any), - "createdAt": llx.TimeDataPtr(pr.CreatedAt), + "createdAt": llx.TimeDataPtr(githubTimestamp(pr.CreatedAt)), "title": llx.StringDataPtr(pr.Title), "owner": llx.ResourceData(owner, owner.MqlName()), "assignees": llx.ArrayData(assigneesRes, types.Any), @@ -1475,9 +1475,9 @@ func (g *mqlGithubRepository) getIssues(state string) ([]interface{}, error) { "state": llx.StringData(issue.GetState()), "body": llx.StringData(issue.GetBody()), "url": llx.StringData(issue.GetURL()), - "createdAt": llx.TimeDataPtr(issue.CreatedAt), - "updatedAt": llx.TimeDataPtr(issue.UpdatedAt), - "closedAt": llx.TimeDataPtr(issue.ClosedAt), + "createdAt": llx.TimeDataPtr(githubTimestamp(issue.CreatedAt)), + "updatedAt": llx.TimeDataPtr(githubTimestamp(issue.UpdatedAt)), + "closedAt": llx.TimeDataPtr(githubTimestamp(issue.ClosedAt)), "assignees": llx.ArrayData(assignees, types.Any), "closedBy": llx.AnyData(closedBy), }) diff --git a/providers/github/resources/github_team.go b/providers/github/resources/github_team.go index 1b40cb0cd6..12ba7714e9 100644 --- a/providers/github/resources/github_team.go +++ b/providers/github/resources/github_team.go @@ -8,7 +8,7 @@ import ( "strconv" "strings" - "github.com/google/go-github/v49/github" + "github.com/google/go-github/v55/github" "go.mondoo.com/cnquery/llx" "go.mondoo.com/cnquery/providers/github/connection" ) diff --git a/providers/github/resources/github_user.go b/providers/github/resources/github_user.go index 4979d3ec8a..7add44fe5c 100644 --- a/providers/github/resources/github_user.go +++ b/providers/github/resources/github_user.go @@ -11,7 +11,7 @@ import ( "time" "github.com/cockroachdb/errors" - "github.com/google/go-github/v49/github" + "github.com/google/go-github/v55/github" "go.mondoo.com/cnquery/llx" "go.mondoo.com/cnquery/providers-sdk/v1/plugin" "go.mondoo.com/cnquery/providers/github/connection" @@ -204,8 +204,8 @@ func (g *mqlGithubUser) gists() ([]interface{}, error) { r, err := CreateResource(g.MqlRuntime, "github.gist", map[string]*llx.RawData{ "id": llx.StringDataPtr(gist.ID), "description": llx.StringDataPtr(gist.Description), - "createdAt": llx.TimeDataPtr(gist.CreatedAt), - "updatedAt": llx.TimeDataPtr(gist.UpdatedAt), + "createdAt": llx.TimeDataPtr(githubTimestamp(gist.CreatedAt)), + "updatedAt": llx.TimeDataPtr(githubTimestamp(gist.UpdatedAt)), "public": llx.BoolDataPtr(gist.Public), "owner": llx.ResourceData(g, g.MqlName()), "files": llx.ArrayData(files, types.Any), diff --git a/providers/github/resources/github_workflow.go b/providers/github/resources/github_workflow.go index 9e8bc62815..40e1e4e6c9 100644 --- a/providers/github/resources/github_workflow.go +++ b/providers/github/resources/github_workflow.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/google/go-github/v49/github" + "github.com/google/go-github/v55/github" "go.mondoo.com/cnquery/providers-sdk/v1/util/convert" "go.mondoo.com/cnquery/providers/github/connection" "sigs.k8s.io/yaml"