diff --git a/internal/datalakes/inmemory/inmemory.go b/internal/datalakes/inmemory/inmemory.go index a21a68f256..96edb91b80 100644 --- a/internal/datalakes/inmemory/inmemory.go +++ b/internal/datalakes/inmemory/inmemory.go @@ -13,7 +13,7 @@ import ( // Db is the database backend, it allows the interaction with the underlying data. type Db struct { - cache kvStore + cache KVStore services *explorer.LocalServices // bidirectional connection between db + services uuid string // used for all object identifiers to prevent clashes (eg in-memory pubsub) nowProvider func() time.Time @@ -21,7 +21,7 @@ type Db struct { // NewServices creates a new set of backend services func NewServices(runtime llx.Runtime) (*Db, *explorer.LocalServices, error) { - var cache kvStore = newKissDb() + var cache KVStore = NewKissDb() db := &Db{ cache: cache, diff --git a/internal/datalakes/inmemory/kiss.go b/internal/datalakes/inmemory/kiss.go index 3468f8ade1..a4bbad3503 100644 --- a/internal/datalakes/inmemory/kiss.go +++ b/internal/datalakes/inmemory/kiss.go @@ -5,8 +5,8 @@ package inmemory import "sync" -// kvStore is an general-purpose abstraction for key-value stores -type kvStore interface { +// KVStore is an general-purpose abstraction for key-value stores +type KVStore interface { Get(key interface{}) (interface{}, bool) Set(key interface{}, value interface{}, cost int64) bool Del(key interface{}) @@ -18,7 +18,7 @@ type kissDb struct { data map[string]interface{} } -func newKissDb() *kissDb { +func NewKissDb() *kissDb { return &kissDb{ data: map[string]interface{}{}, } diff --git a/providers-sdk/v1/plugin/service.go b/providers-sdk/v1/plugin/service.go index ea36c6f9a1..94dedb7119 100644 --- a/providers-sdk/v1/plugin/service.go +++ b/providers-sdk/v1/plugin/service.go @@ -11,6 +11,7 @@ import ( sync "sync" "time" + "go.mondoo.com/cnquery/v11/internal/datalakes/inmemory" llx "go.mondoo.com/cnquery/v11/llx" inventory "go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory" ) @@ -24,11 +25,14 @@ type Service struct { lastHeartbeat int64 heartbeatLock sync.Mutex + + Cache inmemory.KVStore } func NewService() *Service { return &Service{ runtimes: make(map[uint32]*Runtime), + Cache: inmemory.NewKissDb(), } } diff --git a/providers/github/connection/connection.go b/providers/github/connection/connection.go index c974ef0b06..a8e1274d85 100644 --- a/providers/github/connection/connection.go +++ b/providers/github/connection/connection.go @@ -15,6 +15,7 @@ import ( "github.com/cockroachdb/errors" "github.com/google/go-github/v67/github" "github.com/hashicorp/go-retryablehttp" + "github.com/mitchellh/hashstructure/v2" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "go.mondoo.com/cnquery/v11/logger/zerologadapter" @@ -38,6 +39,9 @@ type GithubConnection struct { asset *inventory.Asset client *github.Client ctx context.Context + + // Used to avoid verifying a client with the same options more than once + Hash uint64 } func NewGithubConnection(id uint32, asset *inventory.Asset) (*GithubConnection, error) { @@ -73,20 +77,20 @@ func NewGithubConnection(id uint32, asset *inventory.Asset) (*GithubConnection, // (default behaviour is to send fake 403 response bypassing the retry logic) ctx := context.WithValue(context.Background(), github.SleepUntilPrimaryRateLimitResetWhenRateLimited, true) - // perform a quick call to verify the token's validity. - // @afiune do we need to validate the token for every connection? can this be a "once" operation? - _, resp, err := client.Meta.Zen(ctx) + // store the hash of the config options used to generate this client + hash, err := hashstructure.Hash(conf.Options, hashstructure.FormatV2, nil) if err != nil { - if resp != nil && resp.StatusCode == 401 { - return nil, errors.New("invalid GitHub token provided. check the value passed with the --token flag or the GITHUB_TOKEN environment variable") - } - return nil, err + // not a blocker since this is only used to avoid validating + // the client multiple times + log.Warn().Err(err).Msg("unable to hash config options") } + return &GithubConnection{ Connection: plugin.NewConnection(id, asset), asset: asset, client: client, ctx: ctx, + Hash: hash, }, nil } @@ -106,6 +110,20 @@ func (c *GithubConnection) Context() context.Context { return c.ctx } +func (c *GithubConnection) Verify() error { + // perform a quick call to verify the token's validity. + _, resp, err := c.client.Meta.Zen(c.ctx) + if err != nil { + if resp != nil && resp.StatusCode == 401 { + return errors.New( + "invalid GitHub token provided. check the value passed with the --token flag or the GITHUB_TOKEN environment variable", + ) + } + return err + } + return nil +} + func newGithubAppClient(conf *inventory.Config) (*github.Client, error) { appIdStr := conf.Options[OPTION_APP_ID] if appIdStr == "" { diff --git a/providers/github/provider/provider.go b/providers/github/provider/provider.go index 25cdfdeaa1..fa99a3b23a 100644 --- a/providers/github/provider/provider.go +++ b/providers/github/provider/provider.go @@ -163,6 +163,15 @@ func (s *Service) connect(req *plugin.ConnectReq, callback plugin.ProviderCallba return nil, err } + if _, ok := s.Cache.Get(conn.Hash); !ok { + // verify the connection only once + if err := conn.Verify(); err != nil { + return nil, err + } + // store the hash of the connection + s.Cache.Set(conn.Hash, true, 1) + } + var upstream *upstream.UpstreamClient if req.Upstream != nil && !req.Upstream.Incognito { upstream, err = req.Upstream.InitClient(context.Background())