⭐️ plugin cache + verify github conn once

Signed-off-by: Salim Afiune Maya <[email protected]>

internal/datalakes/inmemory/inmemory.go

@@ -13,15 +13,15 @@ import (
 
 // Db is the database backend, it allows the interaction with the underlying data.
 type Db struct {
-	cache       kvStore
+	cache       KVStore
 	services    *explorer.LocalServices // bidirectional connection between db + services
 	uuid        string                  // used for all object identifiers to prevent clashes (eg in-memory pubsub)
 	nowProvider func() time.Time
 }
 
 // NewServices creates a new set of backend services
 func NewServices(runtime llx.Runtime) (*Db, *explorer.LocalServices, error) {
-	var cache kvStore = newKissDb()
+	var cache KVStore = NewKissDb()
 
 	db := &Db{
 		cache:       cache,

internal/datalakes/inmemory/kiss.go

@@ -5,8 +5,8 @@ package inmemory
 
 import "sync"
 
-// kvStore is an general-purpose abstraction for key-value stores
-type kvStore interface {
+// KVStore is an general-purpose abstraction for key-value stores
+type KVStore interface {
 	Get(key interface{}) (interface{}, bool)
 	Set(key interface{}, value interface{}, cost int64) bool
 	Del(key interface{})
@@ -18,7 +18,7 @@ type kissDb struct {
 	data map[string]interface{}
 }
 
-func newKissDb() *kissDb {
+func NewKissDb() *kissDb {
 	return &kissDb{
 		data: map[string]interface{}{},
 	}

providers-sdk/v1/plugin/service.go

@@ -11,6 +11,7 @@ import (
 	sync "sync"
 	"time"
 
+	"go.mondoo.com/cnquery/v11/internal/datalakes/inmemory"
 	llx "go.mondoo.com/cnquery/v11/llx"
 	inventory "go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
 )
@@ -24,11 +25,14 @@ type Service struct {
 
 	lastHeartbeat int64
 	heartbeatLock sync.Mutex
+
+	Cache inmemory.KVStore
 }
 
 func NewService() *Service {
 	return &Service{
 		runtimes: make(map[uint32]*Runtime),
+		Cache:    inmemory.NewKissDb(),
 	}
 }
 

providers/github/connection/connection.go

@@ -15,6 +15,7 @@ import (
 	"github.com/cockroachdb/errors"
 	"github.com/google/go-github/v67/github"
 	"github.com/hashicorp/go-retryablehttp"
+	"github.com/mitchellh/hashstructure/v2"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/v11/logger/zerologadapter"
@@ -38,6 +39,9 @@ type GithubConnection struct {
 	asset  *inventory.Asset
 	client *github.Client
 	ctx    context.Context
+
+	// Used to avoid verifying a client with the same options more than once
+	Hash uint64
 }
 
 func NewGithubConnection(id uint32, asset *inventory.Asset) (*GithubConnection, error) {
@@ -73,20 +77,20 @@ func NewGithubConnection(id uint32, asset *inventory.Asset) (*GithubConnection,
 	// (default behaviour is to send fake 403 response bypassing the retry logic)
 	ctx := context.WithValue(context.Background(), github.SleepUntilPrimaryRateLimitResetWhenRateLimited, true)
 
-	// perform a quick call to verify the token's validity.
-	// @afiune do we need to validate the token for every connection? can this be a "once" operation?
-	_, resp, err := client.Meta.Zen(ctx)
+	// store the hash of the config options used to generate this client
+	hash, err := hashstructure.Hash(conf.Options, hashstructure.FormatV2, nil)
 	if err != nil {
-		if resp != nil && resp.StatusCode == 401 {
-			return nil, errors.New("invalid GitHub token provided. check the value passed with the --token flag or the GITHUB_TOKEN environment variable")
-		}
-		return nil, err
+		// not a blocker since this is only used to avoid validating
+		// the client multiple times
+		log.Warn().Err(err).Msg("unable to hash config options")
 	}
+
 	return &GithubConnection{
 		Connection: plugin.NewConnection(id, asset),
 		asset:      asset,
 		client:     client,
 		ctx:        ctx,
+		Hash:       hash,
 	}, nil
 }
 
@@ -106,6 +110,20 @@ func (c *GithubConnection) Context() context.Context {
 	return c.ctx
 }
 
+func (c *GithubConnection) Verify() error {
+	// perform a quick call to verify the token's validity.
+	_, resp, err := c.client.Meta.Zen(c.ctx)
+	if err != nil {
+		if resp != nil && resp.StatusCode == 401 {
+			return errors.New(
+				"invalid GitHub token provided. check the value passed with the --token flag or the GITHUB_TOKEN environment variable",
+			)
+		}
+		return err
+	}
+	return nil
+}
+
 func newGithubAppClient(conf *inventory.Config) (*github.Client, error) {
 	appIdStr := conf.Options[OPTION_APP_ID]
 	if appIdStr == "" {

providers/github/provider/provider.go

@@ -163,6 +163,15 @@ func (s *Service) connect(req *plugin.ConnectReq, callback plugin.ProviderCallba
 			return nil, err
 		}
 
+		if _, ok := s.Cache.Get(conn.Hash); !ok {
+			// verify the connection only once
+			if err := conn.Verify(); err != nil {
+				return nil, err
+			}
+			// store the hash of the connection
+			s.Cache.Set(conn.Hash, true, 1)
+		}
+
 		var upstream *upstream.UpstreamClient
 		if req.Upstream != nil && !req.Upstream.Incognito {
 			upstream, err = req.Upstream.InitClient(context.Background())