Update GitHub library to v55

There's a very large number of improvements here including handling of secondary API limits Signed-off-by: Tim Smith <[email protected]>
mondoohq · Sep 21, 2023 · 307d150 · 307d150
1 parent 8e36364
commit 307d150
Show file tree

Hide file tree

Showing 12 changed files with 42 additions and 32 deletions.
diff --git a/providers/github/connection/connection.go b/providers/github/connection/connection.go
@@ -9,7 +9,7 @@ import (
 	"os"
 
 	"github.com/cockroachdb/errors"
-	"github.com/google/go-github/v49/github"
+	"github.com/google/go-github/v55/github"
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/providers-sdk/v1/inventory"
 	"go.mondoo.com/cnquery/providers-sdk/v1/vault"

diff --git a/providers/github/connection/connection_test.go b/providers/github/connection/connection_test.go
@@ -11,7 +11,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/google/go-github/v49/github"
+	"github.com/google/go-github/v55/github"
 	"github.com/stretchr/testify/require"
 	"go.mondoo.com/cnquery/providers-sdk/v1/inventory"
 )

diff --git a/providers/github/connection/platform.go b/providers/github/connection/platform.go
@@ -7,7 +7,7 @@ import (
 	"context"
 
 	"github.com/cockroachdb/errors"
-	"github.com/google/go-github/v49/github"
+	"github.com/google/go-github/v55/github"
 	"go.mondoo.com/cnquery/providers-sdk/v1/inventory"
 )
 

diff --git a/providers/github/go.mod b/providers/github/go.mod
@@ -6,10 +6,10 @@ go 1.20
 
 require (
 	github.com/cockroachdb/errors v1.11.1
-	github.com/google/go-github/v49 v49.1.0
+	github.com/google/go-github/v55 v55.0.0
 	github.com/rs/zerolog v1.30.0
 	github.com/stretchr/testify v1.8.4
-	go.mondoo.com/cnquery v0.0.0-20230915180754-c5f61bc705cf
+	go.mondoo.com/cnquery v0.0.0-20230921152023-8e36364513b6
 	go.mondoo.com/ranger-rpc v0.5.1
 	golang.org/x/oauth2 v0.12.0
 	sigs.k8s.io/yaml v1.3.0
@@ -86,6 +86,7 @@ require (
 	github.com/docker/docker-credential-helpers v0.8.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/esimonov/ifshort v1.0.4 // indirect
 	github.com/ettle/strcase v0.1.1 // indirect
 	github.com/fatih/color v1.15.0 // indirect
@@ -94,6 +95,7 @@ require (
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/fzipp/gocyclo v0.6.0 // indirect
 	github.com/getsentry/sentry-go v0.24.1 // indirect
+	github.com/glebarez/go-sqlite v1.21.2 // indirect
 	github.com/go-critic/go-critic v0.9.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
@@ -126,7 +128,6 @@ require (
 	github.com/google/go-containerregistry v0.16.1 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
 	github.com/google/uuid v1.3.1 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/gordonklaus/ineffassign v0.0.0-20230610083614-0e73809eb601 // indirect
@@ -271,7 +272,7 @@ require (
 	golang.org/x/tools v0.13.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect
 	google.golang.org/grpc v1.58.1 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
@@ -288,6 +289,10 @@ require (
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kubelet v0.28.2 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
+	modernc.org/libc v1.24.1 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.1 // indirect
+	modernc.org/sqlite v1.25.0 // indirect
 	moul.io/http2curl v1.0.0 // indirect
 	mvdan.cc/gofumpt v0.5.0 // indirect
 	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect

diff --git a/providers/github/go.sum b/providers/github/go.sum
@@ -208,6 +208,7 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -232,7 +233,8 @@ github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
 github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
 github.com/getsentry/sentry-go v0.24.1 h1:W6/0GyTy8J6ge6lVCc94WB6Gx2ZuLrgopnn9w8Hiwuk=
 github.com/getsentry/sentry-go v0.24.1/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
-github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4=
+github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
+github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
 github.com/go-critic/go-critic v0.9.0 h1:Pmys9qvU3pSML/3GEQ2Xd9RZ/ip+aXHKILuxczKGV/U=
 github.com/go-critic/go-critic v0.9.0/go.mod h1:5P8tdXL7m/6qnyG6oRAlYLORvoXH0WDypYgAEmagT40=
 github.com/go-errors/errors v1.5.0 h1:/EuijeGOu7ckFxzhkj4CXJ8JaenxK7bKUxpPYqeLHqQ=
@@ -354,8 +356,8 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ=
 github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
-github.com/google/go-github/v49 v49.1.0 h1:LFkMgawGQ8dfzWLH/rNE0b3u1D3n6/dw7ZmrN3b+YFY=
-github.com/google/go-github/v49 v49.1.0/go.mod h1:MUUzHPrhGniB6vUKa27y37likpipzG+BXXJbG04J334=
+github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
+github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -375,7 +377,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
-github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
@@ -1153,8 +1154,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb h1:Isk1sSH7bovx8Rti2wZK0UZF6oraBDK74uoyLEEVFN0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1239,9 +1240,13 @@ k8s.io/kubelet v0.28.2/go.mod h1:rvd0e7T5TjPcfZvy62P90XhFzp0IhPIOy+Pqy3Rtipo=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM=
+modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak=
 modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
+modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
 modernc.org/memory v1.7.1 h1:9J+2/GKTlV503mk3yv8QJ6oEpRCUrRy0ad8TXEPoV8M=
+modernc.org/memory v1.7.1/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
 modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA=
+modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU=
 moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8=
 moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE=
 mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E=

diff --git a/providers/github/resources/git.go b/providers/github/resources/git.go
@@ -4,7 +4,7 @@
 package resources
 
 import (
-	"github.com/google/go-github/v49/github"
+	"github.com/google/go-github/v55/github"
 	"go.mondoo.com/cnquery/llx"
 	"go.mondoo.com/cnquery/providers-sdk/v1/plugin"
 )
@@ -33,7 +33,7 @@ func newMqlGitAuthor(runtime *plugin.Runtime, sha string, a *github.CommitAuthor
 		"sha":   llx.StringData(sha),
 		"name":  llx.StringData(a.GetName()),
 		"email": llx.StringData(a.GetEmail()),
-		"date":  llx.TimeData(date),
+		"date":  llx.TimeData(githubTimestamp(date)),
 	})
 }
 

diff --git a/providers/github/resources/github.go b/providers/github/resources/github.go
@@ -6,7 +6,7 @@ package resources
 import (
 	"time"
 
-	"github.com/google/go-github/v49/github"
+	"github.com/google/go-github/v55/github"
 )
 
 func githubTimestamp(ts *github.Timestamp) *time.Time {

diff --git a/providers/github/resources/github_org.go b/providers/github/resources/github_org.go
@@ -9,7 +9,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/google/go-github/v49/github"
+	"github.com/google/go-github/v55/github"
 	"go.mondoo.com/cnquery/llx"
 	"go.mondoo.com/cnquery/providers-sdk/v1/plugin"
 	"go.mondoo.com/cnquery/providers-sdk/v1/util/convert"
@@ -43,7 +43,7 @@ func initGithubOrganization(runtime *plugin.Runtime, args map[string]*llx.RawDat
 	if name == "" {
 		name = *org.Name
 	}
-	args["id"] = llx.IntData(convert.ToInt64(org.ID))
+	args["id"] = llx.IntData(org.ID)
 	args["name"] = llx.StringData(name)
 	args["login"] = llx.StringDataPtr(org.Login)
 	args["nodeId"] = llx.StringDataPtr(org.NodeID)
@@ -56,10 +56,10 @@ func initGithubOrganization(runtime *plugin.Runtime, args map[string]*llx.RawDat
 	args["followers"] = llx.IntData(convert.ToInt64FromInt(org.Followers))
 	args["following"] = llx.IntData(convert.ToInt64FromInt(org.Following))
 	args["description"] = llx.StringDataPtr(org.Description)
-	args["createdAt"] = llx.TimeDataPtr(org.CreatedAt)
-	args["updatedAt"] = llx.TimeDataPtr(org.UpdatedAt)
-	args["totalPrivateRepos"] = llx.IntData(convert.ToInt64FromInt(org.TotalPrivateRepos))
-	args["ownedPrivateRepos"] = llx.IntData(convert.ToInt64FromInt(org.OwnedPrivateRepos))
+	args["createdAt"] = llx.TimeDataPtr(githubTimestamp(org.CreatedAt))
+	args["updatedAt"] = llx.TimeDataPtr(githubTimestamp(org.UpdatedAt))
+	args["totalPrivateRepos"] = llx.IntData(org.TotalPrivateRepos)
+	args["ownedPrivateRepos"] = llx.IntData(org.OwnedPrivateRepos)
 	args["privateGists"] = llx.IntData(convert.ToInt64FromInt(org.PrivateGists))
 	args["diskUsage"] = llx.IntData(convert.ToInt64FromInt(org.DiskUsage))
 	args["collaborators"] = llx.IntData(convert.ToInt64FromInt(org.Collaborators))

diff --git a/providers/github/resources/github_repo.go b/providers/github/resources/github_repo.go
@@ -9,7 +9,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/google/go-github/v49/github"
+	"github.com/google/go-github/v55/github"
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/llx"
 	"go.mondoo.com/cnquery/providers-sdk/v1/plugin"
@@ -331,7 +331,7 @@ func (g *mqlGithubRepository) openMergeRequests() ([]interface{}, error) {
 			"number":    llx.IntData(int64(*pr.Number)),
 			"state":     llx.StringDataPtr(pr.State),
 			"labels":    llx.ArrayData(labels, types.Any),
-			"createdAt": llx.TimeDataPtr(pr.CreatedAt),
+			"createdAt": llx.TimeDataPtr(githubTimestamp(pr.CreatedAt)),
 			"title":     llx.StringDataPtr(pr.Title),
 			"owner":     llx.ResourceData(owner, owner.MqlName()),
 			"assignees": llx.ArrayData(assigneesRes, types.Any),
@@ -1475,9 +1475,9 @@ func (g *mqlGithubRepository) getIssues(state string) ([]interface{}, error) {
 			"state":     llx.StringData(issue.GetState()),
 			"body":      llx.StringData(issue.GetBody()),
 			"url":       llx.StringData(issue.GetURL()),
-			"createdAt": llx.TimeDataPtr(issue.CreatedAt),
-			"updatedAt": llx.TimeDataPtr(issue.UpdatedAt),
-			"closedAt":  llx.TimeDataPtr(issue.ClosedAt),
+			"createdAt": llx.TimeDataPtr(githubTimestamp(issue.CreatedAt)),
+			"updatedAt": llx.TimeDataPtr(githubTimestamp(issue.UpdatedAt)),
+			"closedAt":  llx.TimeDataPtr(githubTimestamp(issue.ClosedAt)),
 			"assignees": llx.ArrayData(assignees, types.Any),
 			"closedBy":  llx.AnyData(closedBy),
 		})

diff --git a/providers/github/resources/github_team.go b/providers/github/resources/github_team.go
@@ -8,7 +8,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/google/go-github/v49/github"
+	"github.com/google/go-github/v55/github"
 	"go.mondoo.com/cnquery/llx"
 	"go.mondoo.com/cnquery/providers/github/connection"
 )

diff --git a/providers/github/resources/github_user.go b/providers/github/resources/github_user.go
@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/cockroachdb/errors"
-	"github.com/google/go-github/v49/github"
+	"github.com/google/go-github/v55/github"
 	"go.mondoo.com/cnquery/llx"
 	"go.mondoo.com/cnquery/providers-sdk/v1/plugin"
 	"go.mondoo.com/cnquery/providers/github/connection"
@@ -204,8 +204,8 @@ func (g *mqlGithubUser) gists() ([]interface{}, error) {
 		r, err := CreateResource(g.MqlRuntime, "github.gist", map[string]*llx.RawData{
 			"id":          llx.StringDataPtr(gist.ID),
 			"description": llx.StringDataPtr(gist.Description),
-			"createdAt":   llx.TimeDataPtr(gist.CreatedAt),
-			"updatedAt":   llx.TimeDataPtr(gist.UpdatedAt),
+			"createdAt":   llx.TimeDataPtr(githubTimestamp(gist.CreatedAt)),
+			"updatedAt":   llx.TimeDataPtr(githubTimestamp(gist.UpdatedAt)),
 			"public":      llx.BoolDataPtr(gist.Public),
 			"owner":       llx.ResourceData(g, g.MqlName()),
 			"files":       llx.ArrayData(files, types.Any),

diff --git a/providers/github/resources/github_workflow.go b/providers/github/resources/github_workflow.go
@@ -9,7 +9,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/google/go-github/v49/github"
+	"github.com/google/go-github/v55/github"
 	"go.mondoo.com/cnquery/providers-sdk/v1/util/convert"
 	"go.mondoo.com/cnquery/providers/github/connection"
 	"sigs.k8s.io/yaml"