From ff91734dbbcf3ce991ca7e4ed64b78440dab561b Mon Sep 17 00:00:00 2001
From: Philippe Rolet <pr@dust.tt>
Date: Wed, 20 Nov 2024 14:08:01 +0100
Subject: [PATCH] [Slack] API: fetch public and private remote channels
 separately (#8775)

Description
---
Fixes https://github.com/dust-tt/tasks/issues/1655 (again)

This PR fixes the issues caused by the `conversations.list` rate
limiting

Investigating the above, it was shown (tested on a production
workspace) that fetching both public and private channels at the same
time changes the fetch behavior (likely due to their internal
implementation)
- Fetching both public and private =>  ~400 calls to
`conversations.list` (small pagination batches)
- Fetching public => ~10 calls, then private ~1 call (large pagination
batches)

As such, the PR fetches them separately, which will fix the issue

Risks
---
low: simple, tested change

Deploy
---
connectors
---
 .../src/connectors/slack/temporal/activities.ts  | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/connectors/src/connectors/slack/temporal/activities.ts b/connectors/src/connectors/slack/temporal/activities.ts
index ea62039caf31..0fa6891610a3 100644
--- a/connectors/src/connectors/slack/temporal/activities.ts
+++ b/connectors/src/connectors/slack/temporal/activities.ts
@@ -81,6 +81,20 @@ export const getChannels = cacheWithRedis(
 async function _getChannelsUncached(
   connectorId: ModelId,
   joinedOnly: boolean
+): Promise<Channel[]> {
+  return Promise.all([
+    _getTypedChannelsUncached(connectorId, joinedOnly, "public_channel"),
+    _getTypedChannelsUncached(connectorId, joinedOnly, "private_channel"),
+  ]).then(([publicChannels, privateChannels]) => [
+    ...publicChannels,
+    ...privateChannels,
+  ]);
+}
+
+async function _getTypedChannelsUncached(
+  connectorId: ModelId,
+  joinedOnly: boolean,
+  types: "public_channel" | "private_channel"
 ): Promise<Channel[]> {
   const client = await getSlackClient(connectorId);
   const allChannels = [];
@@ -88,7 +102,7 @@ async function _getChannelsUncached(
   let nbCalls = 0;
   do {
     const c: ConversationsListResponse = await client.conversations.list({
-      types: "public_channel, private_channel",
+      types,
       // despite the limit being 1000, slack may return fewer channels
       // we observed ~50 channels per call at times see https://github.com/dust-tt/tasks/issues/1655
       limit: 999,