From 9deabdb4f01871802c814e989a0aeb0146c16758 Mon Sep 17 00:00:00 2001 From: Jacob Truman Date: Mon, 20 Dec 2021 11:30:36 -0700 Subject: [PATCH 1/7] Migrate from slacker to slack-sdk --- requirements.txt | 2 +- slackbot/slackclient.py | 37 +++++++++++++------------- tests/functional/driver.py | 48 +++++++++++++++++----------------- tests/unit/test_slackclient.py | 4 +-- 4 files changed, 46 insertions(+), 45 deletions(-) diff --git a/requirements.txt b/requirements.txt index 896f094e..19628e61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests>=2.4.0 websocket-client>=0.22.0,<=0.44.0 -slacker>=0.9.50 +slack-sdk>=3.13.0 six>=1.10.0 pytest>=2.9.1 diff --git a/slackbot/slackclient.py b/slackbot/slackclient.py index fb8d5768..b9da2aca 100644 --- a/slackbot/slackclient.py +++ b/slackbot/slackclient.py @@ -7,7 +7,8 @@ import time from ssl import SSLError -import slacker +import slack_sdk + from six import iteritems from websocket import ( @@ -35,16 +36,15 @@ def __init__(self, token, timeout=None, bot_icon=None, bot_emoji=None, connect=T self.rtm_start_args = rtm_start_args if timeout is None: - self.webapi = slacker.Slacker(self.token) + self.webapi = slack_sdk.WebClient(self.token) else: - self.webapi = slacker.Slacker(self.token, timeout=timeout) + self.webapi = slack_sdk.WebClient(self.token, timeout=timeout) if connect: self.rtm_connect() def rtm_connect(self): - reply = self.webapi.rtm.start(**(self.rtm_start_args or {})).body - time.sleep(1) + reply = self.webapi.rtm_connect() self.parse_slack_login_data(reply) def reconnect(self): @@ -61,10 +61,11 @@ def parse_slack_login_data(self, login_data): self.login_data = login_data self.domain = self.login_data['team']['domain'] self.username = self.login_data['self']['name'] - self.parse_user_data(login_data['users']) - self.parse_channel_data(login_data['channels']) - self.parse_channel_data(login_data['groups']) - self.parse_channel_data(login_data['ims']) + self.parse_user_data(self.webapi.users_list()['members']) + self.parse_channel_data(self.webapi.conversations_list( + exclude_archived=True, + types="public_channel,private_channel" + )['channels']) proxy, proxy_port, no_proxy = get_http_proxy(os.environ) @@ -126,26 +127,26 @@ def rtm_send_message(self, channel, message, attachments=None, thread_ts=None): def upload_file(self, channel, fname, fpath, comment): fname = fname or to_utf8(os.path.basename(fpath)) - self.webapi.files.upload(fpath, + self.webapi.files_upload(file=fpath, channels=channel, filename=fname, initial_comment=comment) def upload_content(self, channel, fname, content, comment): - self.webapi.files.upload(None, - channels=channel, + self.webapi.files_upload(channels=channel, content=content, filename=fname, initial_comment=comment) - def send_message(self, channel, message, attachments=None, as_user=True, thread_ts=None): - self.webapi.chat.post_message( - channel, - message, + def send_message(self, channel, message, attachments=None, blocks=None, as_user=True, thread_ts=None): + self.webapi.chat_postMessage( + channel=channel, + text=message, username=self.login_data['self']['name'], icon_url=self.bot_icon, icon_emoji=self.bot_emoji, attachments=attachments, + blocks=blocks, as_user=as_user, thread_ts=thread_ts) @@ -153,7 +154,7 @@ def get_channel(self, channel_id): return Channel(self, self.channels[channel_id]) def open_dm_channel(self, user_id): - return self.webapi.im.open(user_id).body["channel"]["id"] + return self.webapi.conversations_open(users=[user_id])["channel"]["id"] def find_channel_by_name(self, channel_name): for channel_id, channel in iteritems(self.channels): @@ -173,7 +174,7 @@ def find_user_by_name(self, username): return userid def react_to_message(self, emojiname, channel, timestamp): - self.webapi.reactions.add( + self.webapi.reactions_add( name=emojiname, channel=channel, timestamp=timestamp) diff --git a/tests/functional/driver.py b/tests/functional/driver.py index 48381b85..d3584a2e 100644 --- a/tests/functional/driver.py +++ b/tests/functional/driver.py @@ -2,7 +2,7 @@ import json import re import time -import slacker +import slack_sdk import websocket import six from six.moves import _thread, range @@ -13,7 +13,7 @@ class Driver(object): the tests code can concentrate on higher level logic. """ def __init__(self, driver_apitoken, driver_username, testbot_username, channel, private_channel): - self.slacker = slacker.Slacker(driver_apitoken) + self.webapi = slack_sdk.WebClient(driver_apitoken) self.driver_username = driver_username self.driver_userid = None self.test_channel = channel @@ -36,7 +36,7 @@ def start(self): self._rtm_connect() # self._fetch_users() self._start_dm_channel() - self._join_test_channel() + # self._join_test_channel() # the underlying api method is no longer available def wait_for_bot_online(self): self._wait_for_bot_presense(True) @@ -142,7 +142,7 @@ def ensure_reaction_posted(self, emojiname, maxwait=5): def _send_message_to_bot(self, channel, msg): self.clear_events() self._start_ts = time.time() - self.slacker.chat.post_message(channel, msg, username=self.driver_username) + self.webapi.chat_postMessage(channel=channel, text=msg, username=self.driver_username) def _wait_for_bot_message(self, channel, match, maxwait=60, tosender=True, thread=False): for _ in range(maxwait): @@ -157,10 +157,10 @@ def _has_got_message(self, channel, match, start=None, end=None): match = six.text_type(r'\<@{}\>: {}').format(self.driver_userid, match) oldest = start or self._start_ts latest = end or time.time() - func = self.slacker.channels.history if channel.startswith('C') \ - else self.slacker.im.history + func = self.webapi.channels_history if channel.startswith('C') \ + else self.webapi.im_history response = func(channel, oldest=oldest, latest=latest) - for msg in response.body['messages']: + for msg in response['messages']: if msg['type'] == 'message' and re.match(match, msg['text'], re.DOTALL): return True return False @@ -179,19 +179,19 @@ def _has_got_message_rtm(self, channel, match, tosender=True, thread=False): return False def _fetch_users(self): - response = self.slacker.users.list() - for user in response.body['members']: + response = self.webapi.users_list() + for user in response['members']: self.users[user['name']] = user['id'] self.testbot_userid = self.users[self.testbot_username] self.driver_userid = self.users[self.driver_username] def _rtm_connect(self): - r = self.slacker.rtm.start().body + r = self.webapi.rtm_connect() self.driver_username = r['self']['name'] self.driver_userid = r['self']['id'] - self.users = {u['name']: u['id'] for u in r['users']} + self.users = {u['name']: u['id'] for u in self.webapi.users_list()['members']} self.testbot_userid = self.users[self.testbot_username] self._websocket = websocket.create_connection(r['url']) @@ -217,18 +217,18 @@ def _rtm_read_forever(self): def _start_dm_channel(self): """Start a slack direct messages channel with the test bot""" - response = self.slacker.im.open(self.testbot_userid) - self.dm_chan = response.body['channel']['id'] + response = self.webapi.conversations_open(users=[self.testbot_userid]) + self.dm_chan = response['channel']['id'] def _is_testbot_online(self): - response = self.slacker.users.get_presence(self.testbot_userid) - return response.body['presence'] == self.slacker.presence.ACTIVE + response = self.webapi.users_getPresence(user=self.testbot_userid) + return response['presence'] == 'active' def _has_uploaded_file(self, name, start=None, end=None): ts_from = start or self._start_ts ts_to = end or time.time() - response = self.slacker.files.list(user=self.testbot_userid, ts_from=ts_from, ts_to=ts_to) - for f in response.body['files']: + response = self.webapi.files_list(user=self.testbot_userid, ts_from=ts_from, ts_to=ts_to) + for f in response['files']: if f['name'] == name: return True return False @@ -248,18 +248,18 @@ def _has_reacted(self, emojiname): for event in self.events: if event['type'] == 'reaction_added' \ and event['user'] == self.testbot_userid \ - and (event.get('reaction', '') == emojiname \ + and (event.get('reaction', '') == emojiname or event.get('name', '') == emojiname): return True return False def _join_test_channel(self): - response = self.slacker.channels.join(self.test_channel) - self.cm_chan = response.body['channel']['id'] + response = self.webapi.channels_join(name=self.test_channel) + self.cm_chan = response['channel']['id'] self._invite_testbot_to_channel() # Slacker/Slack API's still references to private_channels as 'groups' - private_channels = self.slacker.groups.list(self.test_private_channel).body['groups'] + private_channels = self.webapi.groups_list(self.test_private_channel)['groups'] for private_channel in private_channels: if self.test_private_channel == private_channel['name']: self.gm_chan = private_channel['id'] @@ -270,12 +270,12 @@ def _join_test_channel(self): self.test_private_channel)) def _invite_testbot_to_channel(self): - if self.testbot_userid not in self.slacker.channels.info(self.cm_chan).body['channel']['members']: - self.slacker.channels.invite(self.cm_chan, self.testbot_userid) + if self.testbot_userid not in self.webapi.channels_info(channel=self.cm_chan)['channel']['members']: + self.webapi.channels_invite(channel=self.cm_chan, user=self.testbot_userid) def _invite_testbot_to_private_channel(self, private_channel): if self.testbot_userid not in private_channel['members']: - self.slacker.groups.invite(self.gm_chan, self.testbot_userid) + self.webapi.groups_invite(channel=self.gm_chan, user=self.testbot_userid) def _is_bot_message(self, msg): if msg['type'] != 'message': diff --git a/tests/unit/test_slackclient.py b/tests/unit/test_slackclient.py index 580f10ed..b616a9d5 100644 --- a/tests/unit/test_slackclient.py +++ b/tests/unit/test_slackclient.py @@ -80,8 +80,8 @@ def test_parse_user_data(slack_client): def test_init_with_timeout(): client = SlackClient(None, connect=False) - assert client.webapi.api.timeout == 10 # seconds default timeout + assert client.webapi.timeout == 30 # seconds default timeout expected_timeout = 42 # seconds client = SlackClient(None, connect=False, timeout=expected_timeout) - assert client.webapi.api.timeout == expected_timeout + assert client.webapi.timeout == expected_timeout From 0afff163eac55522e9bb5623172212a6ccf62b45 Mon Sep 17 00:00:00 2001 From: Jacob Truman Date: Thu, 23 Dec 2021 09:30:22 -0700 Subject: [PATCH 2/7] Update dependencies in setup.py --- .gitignore | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index bdfcd70e..632876a6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ slackbot_test_settings.py /*.egg-info .cache /.vscode/ +.idea \ No newline at end of file diff --git a/setup.py b/setup.py index 2c38344d..710a56d6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ install_requires = ( 'requests>=2.4.0', 'websocket-client>=0.22.0,<=0.44.0', - 'slacker>=0.9.50', + 'slack-sdk>=3.13.0', 'six>=1.10.0' ) # yapf: disable From 8ea3619d1d6b1980ce0d05c310102ad92e5fbca3 Mon Sep 17 00:00:00 2001 From: Jacob Truman Date: Thu, 23 Dec 2021 18:52:55 -0700 Subject: [PATCH 3/7] Leverage pagination for users and channels --- slackbot/slackclient.py | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/slackbot/slackclient.py b/slackbot/slackclient.py index b9da2aca..4ba84fc5 100644 --- a/slackbot/slackclient.py +++ b/slackbot/slackclient.py @@ -8,6 +8,7 @@ from ssl import SSLError import slack_sdk +from slack_sdk.errors import SlackApiError from six import iteritems @@ -61,11 +62,8 @@ def parse_slack_login_data(self, login_data): self.login_data = login_data self.domain = self.login_data['team']['domain'] self.username = self.login_data['self']['name'] - self.parse_user_data(self.webapi.users_list()['members']) - self.parse_channel_data(self.webapi.conversations_list( - exclude_archived=True, - types="public_channel,private_channel" - )['channels']) + self.get_user_data() + self.get_channel_data() proxy, proxy_port, no_proxy = get_http_proxy(os.environ) @@ -74,9 +72,42 @@ def parse_slack_login_data(self, login_data): self.websocket.sock.setblocking(0) + def get_channel_data(self): + cursor = True + while cursor: + try: + page = self.webapi.conversations_list(exclude_archived=True, + types="public_channel,private_channel", + limit=200, + cursor=cursor if cursor is not True else None) + self.parse_channel_data(page['channels']) + cursor = page["response_metadata"]["next_cursor"] + except SlackApiError as err: + # catch rate limit errors + if err.response["error"] == "ratelimited": + logger.warning('slackapi rate limit; sleeping 30 seconds and trying again') + time.sleep(30) + else: + raise err + def parse_channel_data(self, channel_data): self.channels.update({c['id']: c for c in channel_data}) + def get_user_data(self): + cursor = True + while cursor: + try: + page = self.webapi.users_list(limit=200, cursor=cursor if cursor is not True else None) + self.parse_user_data(page['members']) + cursor = page["response_metadata"]["next_cursor"] + except SlackApiError as err: + # catch rate limit errors + if err.response["error"] == "ratelimited": + logger.warning('slackapi rate limit; sleeping 30 seconds and trying again') + time.sleep(30) + else: + raise err + def parse_user_data(self, user_data): self.users.update({u['id']: u for u in user_data}) From 761cb0c7a451f6c537ed61ed21800ee62aed5753 Mon Sep 17 00:00:00 2001 From: Jacob Truman Date: Tue, 4 Jan 2022 11:14:53 -0700 Subject: [PATCH 4/7] Leverage built in rate limit retry handler --- slackbot/slackclient.py | 57 ++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 38 deletions(-) diff --git a/slackbot/slackclient.py b/slackbot/slackclient.py index 4ba84fc5..8c9a808e 100644 --- a/slackbot/slackclient.py +++ b/slackbot/slackclient.py @@ -8,7 +8,7 @@ from ssl import SSLError import slack_sdk -from slack_sdk.errors import SlackApiError +from slack_sdk.http_retry.builtin_handlers import RateLimitErrorRetryHandler from six import iteritems @@ -41,6 +41,10 @@ def __init__(self, token, timeout=None, bot_icon=None, bot_emoji=None, connect=T else: self.webapi = slack_sdk.WebClient(self.token, timeout=timeout) + rate_limit_handler = RateLimitErrorRetryHandler(max_retry_count=100) + # Enable rate limited error retries + self.webapi.retry_handlers.append(rate_limit_handler) + if connect: self.rtm_connect() @@ -62,52 +66,29 @@ def parse_slack_login_data(self, login_data): self.login_data = login_data self.domain = self.login_data['team']['domain'] self.username = self.login_data['self']['name'] - self.get_user_data() - self.get_channel_data() + for page in self.webapi.users_list(limit=200): + self.parse_user_data(page['members']) + for page in self.webapi.conversations_list( + exclude_archived=True, + types="public_channel,private_channel", + limit=200 + ): + self.parse_channel_data(page['channels']) proxy, proxy_port, no_proxy = get_http_proxy(os.environ) - self.websocket = create_connection(self.login_data['url'], http_proxy_host=proxy, - http_proxy_port=proxy_port, http_no_proxy=no_proxy) + self.websocket = create_connection( + self.login_data['url'], + http_proxy_host=proxy, + http_proxy_port=proxy_port, + http_no_proxy=no_proxy + ) self.websocket.sock.setblocking(0) - def get_channel_data(self): - cursor = True - while cursor: - try: - page = self.webapi.conversations_list(exclude_archived=True, - types="public_channel,private_channel", - limit=200, - cursor=cursor if cursor is not True else None) - self.parse_channel_data(page['channels']) - cursor = page["response_metadata"]["next_cursor"] - except SlackApiError as err: - # catch rate limit errors - if err.response["error"] == "ratelimited": - logger.warning('slackapi rate limit; sleeping 30 seconds and trying again') - time.sleep(30) - else: - raise err - def parse_channel_data(self, channel_data): self.channels.update({c['id']: c for c in channel_data}) - def get_user_data(self): - cursor = True - while cursor: - try: - page = self.webapi.users_list(limit=200, cursor=cursor if cursor is not True else None) - self.parse_user_data(page['members']) - cursor = page["response_metadata"]["next_cursor"] - except SlackApiError as err: - # catch rate limit errors - if err.response["error"] == "ratelimited": - logger.warning('slackapi rate limit; sleeping 30 seconds and trying again') - time.sleep(30) - else: - raise err - def parse_user_data(self, user_data): self.users.update({u['id']: u for u in user_data}) From e9ec1e8466522468c5104bf821587bfbe168572f Mon Sep 17 00:00:00 2001 From: Jacob Truman Date: Wed, 5 Jan 2022 10:07:21 -0700 Subject: [PATCH 5/7] Add ping to channel and user add, to prevent websocket timeout/disconnect --- slackbot/slackclient.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/slackbot/slackclient.py b/slackbot/slackclient.py index 8c9a808e..15d4ea63 100644 --- a/slackbot/slackclient.py +++ b/slackbot/slackclient.py @@ -51,6 +51,7 @@ def __init__(self, token, timeout=None, bot_icon=None, bot_emoji=None, connect=T def rtm_connect(self): reply = self.webapi.rtm_connect() self.parse_slack_login_data(reply) + self.connected = True def reconnect(self): while True: @@ -66,14 +67,6 @@ def parse_slack_login_data(self, login_data): self.login_data = login_data self.domain = self.login_data['team']['domain'] self.username = self.login_data['self']['name'] - for page in self.webapi.users_list(limit=200): - self.parse_user_data(page['members']) - for page in self.webapi.conversations_list( - exclude_archived=True, - types="public_channel,private_channel", - limit=200 - ): - self.parse_channel_data(page['channels']) proxy, proxy_port, no_proxy = get_http_proxy(os.environ) @@ -86,10 +79,25 @@ def parse_slack_login_data(self, login_data): self.websocket.sock.setblocking(0) + logger.debug('Getting users') + for page in self.webapi.users_list(limit=200): + self.parse_user_data(page['members']) + logger.debug('Getting channels') + for page in self.webapi.conversations_list( + exclude_archived=True, + types="public_channel,private_channel", + limit=1000 + ): + self.parse_channel_data(page['channels']) + def parse_channel_data(self, channel_data): + self.ping() + logger.debug(f'Adding {len(channel_data)} channels') self.channels.update({c['id']: c for c in channel_data}) def parse_user_data(self, user_data): + self.ping() + logger.debug(f'Adding {len(user_data)} users') self.users.update({u['id']: u for u in user_data}) def send_to_websocket(self, data): From eee07bc039b94dca1a013234f221b608aa90e135 Mon Sep 17 00:00:00 2001 From: Jacob Truman Date: Wed, 5 Jan 2022 14:24:26 -0700 Subject: [PATCH 6/7] Update log message to support python 2 --- slackbot/slackclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slackbot/slackclient.py b/slackbot/slackclient.py index 15d4ea63..1d944a58 100644 --- a/slackbot/slackclient.py +++ b/slackbot/slackclient.py @@ -92,12 +92,12 @@ def parse_slack_login_data(self, login_data): def parse_channel_data(self, channel_data): self.ping() - logger.debug(f'Adding {len(channel_data)} channels') + logger.debug('Adding %d channels', len(channel_data)) self.channels.update({c['id']: c for c in channel_data}) def parse_user_data(self, user_data): self.ping() - logger.debug(f'Adding {len(user_data)} users') + logger.debug('Adding %d users', len(user_data)) self.users.update({u['id']: u for u in user_data}) def send_to_websocket(self, data): From b9257cfce5f76b94f6c60e78b110be4b56660b8e Mon Sep 17 00:00:00 2001 From: Jacob Truman Date: Thu, 6 Jan 2022 16:24:08 -0700 Subject: [PATCH 7/7] =?UTF-8?q?Bump=20version:=201.0.0=20=E2=86=92=201.1.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 3 +-- slackbot/VERSION | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index df96658d..72a664f0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,8 +1,7 @@ [bumpversion] -current_version = 1.0.0 +current_version = 1.1.0 commit = True tag = True tag_name = {new_version} [bumpversion:file:slackbot/VERSION] - diff --git a/slackbot/VERSION b/slackbot/VERSION index 3eefcb9d..9084fa2f 100644 --- a/slackbot/VERSION +++ b/slackbot/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0