Skip to content

Commit

Permalink
🚤 improve performance for large slack accounts
Browse files Browse the repository at this point in the history
If you are a large organisation with 1000s of conversations, the previous implementation fetched information about the creator via the user rest call. The problem is that the slack api heavily rate limits the user rest call.

This change makes the creator information asynchronous and therefore allows much speedier fetching of the conversations. This avoids reaching the rate limit.
  • Loading branch information
chris-rock committed Jan 29, 2024
1 parent 113d7f3 commit c455a5f
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 16 deletions.
38 changes: 38 additions & 0 deletions providers/slack/connection/connection_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Copyright (c) Mondoo, Inc.
// SPDX-License-Identifier: BUSL-1.1

//go:build debugtest
// +build debugtest

package connection

import (
"context"
"testing"

"github.com/slack-go/slack"
"github.com/stretchr/testify/require"
"go.mondoo.com/cnquery/v10/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v10/providers-sdk/v1/vault"
)

func TestSlackProvider(t *testing.T) {

cred := &vault.Credential{
Type: vault.CredentialType_password,
Password: "<slack-token>",
}
cred.PreProcess()

conn, err := NewSlackConnection(1, &inventory.Asset{}, &inventory.Config{
Type: "slack",
Credentials: []*vault.Credential{cred},
})
require.NoError(t, err)

client := conn.Client()
ctx := context.Background()
users, err := client.GetUsersContext(ctx, slack.GetUsersOptionLimit(999))
require.NoError(t, err)
require.NotNil(t, users)
}
4 changes: 4 additions & 0 deletions providers/slack/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/hashicorp/go-retryablehttp v0.7.5
github.com/rs/zerolog v1.31.0
github.com/slack-go/slack v0.12.3
github.com/stretchr/testify v1.8.4
go.mondoo.com/cnquery/v10 v10.0.3
)

Expand Down Expand Up @@ -47,6 +48,7 @@ require (
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/danieljoos/wincred v1.2.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
Expand Down Expand Up @@ -93,6 +95,7 @@ require (
github.com/muesli/termenv v0.15.2 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/rivo/uniseg v0.4.6 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
Expand Down Expand Up @@ -124,6 +127,7 @@ require (
google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe // indirect
google.golang.org/grpc v1.61.0 // indirect
google.golang.org/protobuf v1.32.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
moul.io/http2curl v1.0.0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)
39 changes: 26 additions & 13 deletions providers/slack/resources/conversations.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,21 +118,9 @@ func newMqlSlackConversation(runtime *plugin.Runtime, conversation slack.Channel

created := conversation.Created.Time()

var creator plugin.Resource

if conversation.Creator != "" {
creator, err = NewResource(runtime, "slack.user", map[string]*llx.RawData{
"id": llx.StringData(conversation.Creator),
})
if err != nil {
return nil, err
}
}

return CreateResource(runtime, "slack.conversation", map[string]*llx.RawData{
r, err := CreateResource(runtime, "slack.conversation", map[string]*llx.RawData{
"id": llx.StringData(conversation.ID),
"name": llx.StringData(conversation.Name),
"creator": llx.ResourceData(creator, "slack.user"),
"created": llx.TimeData(created),
"locale": llx.StringData(conversation.Locale),
"topic": llx.DictData(topic),
Expand All @@ -150,12 +138,37 @@ func newMqlSlackConversation(runtime *plugin.Runtime, conversation slack.Channel
"isOrgShared": llx.BoolData(conversation.IsOrgShared),
"priority": llx.FloatData(conversation.Priority),
})
if err != nil {
return nil, err
}
mqlConversation := r.(*mqlSlackConversation)
mqlConversation.creatorId = conversation.Creator
return mqlConversation, nil
}

type mqlSlackConversationInternal struct {
creatorId string
}

func (x *mqlSlackConversation) id() (string, error) {
return "slack.conversation/" + x.Id.Data, nil
}

func (s *mqlSlackConversation) creator() (*mqlSlackUser, error) {
if s.creatorId == "" {
s.Creator = plugin.TValue[*mqlSlackUser]{State: plugin.StateIsSet | plugin.StateIsNull}
return nil, nil
}

r, err := NewResource(s.MqlRuntime, "slack.user", map[string]*llx.RawData{
"id": llx.StringData(s.creatorId),
})
if err != nil {
return nil, err
}
return r.(*mqlSlackUser), nil
}

func (s *mqlSlackConversation) members() ([]interface{}, error) {
conn := s.MqlRuntime.Connection.(*connection.SlackConnection)
client := conn.Client()
Expand Down
2 changes: 1 addition & 1 deletion providers/slack/resources/slack.lr
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ slack.conversation @defaults("id name") {
// Name of the conversation
name string
// User that created this conversation
creator slack.user
creator() slack.user
// Timestamp of when the conversation was created
created time
// IETF language code that represents chosen language
Expand Down
16 changes: 14 additions & 2 deletions providers/slack/resources/slack.lr.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit c455a5f

Please sign in to comment.