From 5544cbaf319f0d729210aa7a06432d9a2d2d31ea Mon Sep 17 00:00:00 2001 From: Ethan <47520067+Skelmis@users.noreply.github.com> Date: Sat, 23 Dec 2023 22:14:06 +1300 Subject: [PATCH 01/22] Revert "chore: version 3.19 (#65)" (#69) This reverts commit 222da50e8b43207a1c646651953854c4f2bd9772. --- suggestions/bot.py | 2 +- suggestions/cogs/help_guild_cog.py | 22 ++++++---------------- suggestions/main.py | 2 +- suggestions/zonis_routes.py | 9 +++++++++ 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/suggestions/bot.py b/suggestions/bot.py index 6ec89c5..58f439a 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -46,7 +46,7 @@ class SuggestionsBot(commands.AutoShardedInteractionBot, BotBase): def __init__(self, *args, **kwargs): - self.version: str = "Public Release 3.19" + self.version: str = "Public Release 3.18" self.main_guild_id: int = 601219766258106399 self.legacy_beta_role_id: int = 995588041991274547 self.automated_beta_role_id: int = 998173237282361425 diff --git a/suggestions/cogs/help_guild_cog.py b/suggestions/cogs/help_guild_cog.py index 55f210e..4b1d602 100644 --- a/suggestions/cogs/help_guild_cog.py +++ b/suggestions/cogs/help_guild_cog.py @@ -76,26 +76,16 @@ async def instance_info( shard_id = self.bot.get_shard_id(guild_id) cluster_id = ( 1 - if shard_id < 5 - else 2 if shard_id < 10 - else 3 - if shard_id < 15 - else 4 + else 2 if shard_id < 20 - else 5 - if shard_id < 25 - else 6 + else 3 if shard_id < 30 - else 7 - if shard_id < 35 - else 8 + else 4 if shard_id < 40 - else 9 - if shard_id < 45 - else 10 + else 5 if shard_id < 50 - else 11 + else 6 ) await interaction.send( @@ -164,7 +154,7 @@ async def show_bot_status( title="Bot infrastructure status", ) down_shards: list[str] = [str(i) for i in range(53)] - down_clusters: list[str] = [str(i) for i in range(1, 12)] + down_clusters: list[str] = [str(i) for i in range(1, 7)] avg_bot_latency: list[float] = [] async with aiohttp.ClientSession( headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]} diff --git a/suggestions/main.py b/suggestions/main.py index 602c79c..f1d6074 100644 --- a/suggestions/main.py +++ b/suggestions/main.py @@ -29,7 +29,7 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot: total_shards = 53 cluster_id = int(os.environ["CLUSTER"]) offset = cluster_id - 1 - number_of_shards_per_cluster = 5 + number_of_shards_per_cluster = 10 shard_ids = [ i for i in range( diff --git a/suggestions/zonis_routes.py b/suggestions/zonis_routes.py index 12d4f5f..5cbf873 100644 --- a/suggestions/zonis_routes.py +++ b/suggestions/zonis_routes.py @@ -79,3 +79,12 @@ async def share_with_devs(self, title, description, sender): ) embed.set_footer(text=f"Sender: {sender}") await channel.send(embed=embed) + + @client.route() + async def refresh_premium(self, user_id: int): + # TODO Implement the ability to refresh premium states for a user + return True + + @client.route() + async def shared_guilds(self, guild_ids: list[int]): + return [gid for gid in guild_ids if gid in self.bot.guild_ids] From 2b32680ab1c1ecb7aedee83f92d3e82db3c9b2cf Mon Sep 17 00:00:00 2001 From: Ethan <47520067+Skelmis@users.noreply.github.com> Date: Thu, 4 Jan 2024 17:54:33 +1300 Subject: [PATCH 02/22] feature branch version 3.20 (#67) * chore: change bot version * fix: viewing voters for cleared suggestions (#70) * feat: add cached counts route * Feat/proxy (#71) * Revert "chore: version 3.19 (#65)" (#69) This reverts commit 222da50e8b43207a1c646651953854c4f2bd9772. * feat: add support for proxy * fix: git being dumb * fix: misc --- suggestions/bot.py | 15 +++++++++++++- suggestions/cogs/view_voters_cog.py | 32 +++++++++++++++++++++++++++++ suggestions/locales/en_GB.json | 3 ++- suggestions/main.py | 5 ++++- suggestions/state.py | 3 ++- suggestions/zonis_routes.py | 24 ++++++++++++++++------ 6 files changed, 72 insertions(+), 10 deletions(-) diff --git a/suggestions/bot.py b/suggestions/bot.py index 58f439a..329f260 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -46,7 +46,7 @@ class SuggestionsBot(commands.AutoShardedInteractionBot, BotBase): def __init__(self, *args, **kwargs): - self.version: str = "Public Release 3.18" + self.version: str = "Public Release 3.20" self.main_guild_id: int = 601219766258106399 self.legacy_beta_role_id: int = 995588041991274547 self.automated_beta_role_id: int = 998173237282361425 @@ -110,6 +110,19 @@ def __init__(self, *args, **kwargs): self.zonis: ZonisRoutes = ZonisRoutes(self) + async def launch_shard( + self, _gateway: str, shard_id: int, *, initial: bool = False + ) -> None: + # Use the proxy if set, else fall back to whatever is default + proxy: Optional[str] = os.environ.get("GW_PROXY", _gateway) + return await super().launch_shard(proxy, shard_id, initial=initial) + + async def before_identify_hook( + self, _shard_id: int | None, *, initial: bool = False # noqa: ARG002 + ) -> None: + # gateway-proxy + return + async def get_or_fetch_channel(self, channel_id: int) -> WrappedChannel: try: return await super().get_or_fetch_channel(channel_id) diff --git a/suggestions/cogs/view_voters_cog.py b/suggestions/cogs/view_voters_cog.py index a184030..36b970b 100644 --- a/suggestions/cogs/view_voters_cog.py +++ b/suggestions/cogs/view_voters_cog.py @@ -12,6 +12,7 @@ from suggestions import Colors from suggestions.cooldown_bucket import InteractionBucket from suggestions.objects import Suggestion +from suggestions.objects.suggestion import SuggestionState if TYPE_CHECKING: from alaric import Document @@ -109,6 +110,13 @@ async def view_suggestion_voters( channel_id=interaction.channel_id, state=self.state, ) + if suggestion.state == SuggestionState.cleared: + return await interaction.send( + self.bot.get_locale( + "VIEW_VOTERS_CLEARED_SUGGESTION", interaction.locale + ), + ephemeral=True, + ) up_vote: disnake.Emoji = await self.bot.suggestion_emojis.default_up_vote() down_vote: disnake.Emoji = await self.bot.suggestion_emojis.default_down_vote() @@ -140,6 +148,14 @@ async def view_suggestion_up_voters( channel_id=interaction.channel_id, state=self.state, ) + if suggestion.state == SuggestionState.cleared: + return await interaction.send( + self.bot.get_locale( + "VIEW_VOTERS_CLEARED_SUGGESTION", interaction.locale + ), + ephemeral=True, + ) + data = [] for voter in suggestion.up_voted_by: data.append(f"<@{voter}>") @@ -165,6 +181,14 @@ async def view_suggestion_down_voters( channel_id=interaction.channel_id, state=self.state, ) + if suggestion.state == SuggestionState.cleared: + return await interaction.send( + self.bot.get_locale( + "VIEW_VOTERS_CLEARED_SUGGESTION", interaction.locale + ), + ephemeral=True, + ) + data = [] for voter in suggestion.down_voted_by: data.append(f"<@{voter}>") @@ -203,6 +227,14 @@ async def view_voters( guild_id=interaction.guild_id, state=self.state, ) + if suggestion.state == SuggestionState.cleared: + return await interaction.send( + self.bot.get_locale( + "VIEW_VOTERS_CLEARED_SUGGESTION", interaction.locale + ), + ephemeral=True, + ) + data = [] up_vote: disnake.Emoji = await self.bot.suggestion_emojis.default_up_vote() down_vote: disnake.Emoji = await self.bot.suggestion_emojis.default_down_vote() diff --git a/suggestions/locales/en_GB.json b/suggestions/locales/en_GB.json index 4d3cbc5..401c717 100644 --- a/suggestions/locales/en_GB.json +++ b/suggestions/locales/en_GB.json @@ -121,5 +121,6 @@ "SUGGESTION_ID_NAME": "suggestion_id", "SUGGESTION_ID_DESCRIPTION": "The suggestions ID you wish to reference.", "USER_ID_NAME": "user_id", - "USER_ID_DESCRIPTION": "The users discord id." + "USER_ID_DESCRIPTION": "The users discord id.", + "VIEW_VOTERS_CLEARED_SUGGESTION": "Cannot view a cleared suggestion." } diff --git a/suggestions/main.py b/suggestions/main.py index f1d6074..f314eb4 100644 --- a/suggestions/main.py +++ b/suggestions/main.py @@ -9,6 +9,7 @@ import textwrap from traceback import format_exception +import aiohttp import cooldowns import disnake from disnake import Locale @@ -26,7 +27,9 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot: is_prod: bool = True if os.environ.get("PROD", None) else False if is_prod: - total_shards = 53 + async with aiohttp.ClientSession() as session: + async with session.get("http://localhost:7878/shard-count") as resp: + total_shards = int(await resp.text()) cluster_id = int(os.environ["CLUSTER"]) offset = cluster_id - 1 number_of_shards_per_cluster = 10 diff --git a/suggestions/state.py b/suggestions/state.py index 8de4b46..befaa6a 100644 --- a/suggestions/state.py +++ b/suggestions/state.py @@ -12,6 +12,7 @@ from alaric import AQ from alaric.comparison import EQ from alaric.logical import AND +from alaric.meta import Negate from alaric.projections import PROJECTION, SHOW from bot_base import NonExistentEntry from bot_base.caches import TimedCache @@ -166,7 +167,7 @@ async def populate_sid_cache(self, guild_id: int) -> list: async def populate_view_voters_cache(self, guild_id: int) -> list: self.view_voters_cache.delete_entry(guild_id) data: List[Dict] = await self.database.suggestions.find_many( - AQ(EQ("guild_id", guild_id)), + AQ(AND(EQ("guild_id", guild_id), Negate(EQ("state", "cleared")))), projections=PROJECTION(SHOW("_id")), try_convert=False, ) diff --git a/suggestions/zonis_routes.py b/suggestions/zonis_routes.py index 5cbf873..5e9b284 100644 --- a/suggestions/zonis_routes.py +++ b/suggestions/zonis_routes.py @@ -37,6 +37,7 @@ def __init__(self, bot: SuggestionsBot): "share_with_devs", "refresh_premium", "shared_guilds", + "cached_item_count", ) async def start(self): @@ -81,10 +82,21 @@ async def share_with_devs(self, title, description, sender): await channel.send(embed=embed) @client.route() - async def refresh_premium(self, user_id: int): - # TODO Implement the ability to refresh premium states for a user - return True + async def cached_item_count(self) -> dict[str, int]: + state = self.bot.state + stats = self.bot.stats + suggestions_queue_cog = self.bot.get_cog("SuggestionsQueueCog") + data = { + "state.autocomplete_cache": len(state.autocomplete_cache), + "state.guild_cache": len(state.guild_cache), + "state.view_voters_cache": len(state.view_voters_cache), + "state.guild_configs": len(state.guild_configs), + "state.user_configs": len(state.user_configs), + "stats.cluster_guild_cache": len(stats.cluster_guild_cache), + "stats.member_stats_cache": len(stats.member_stats_cache), + "suggestions_queue_cog.paginator_objects": len( + suggestions_queue_cog.paginator_objects # noqa + ), + } - @client.route() - async def shared_guilds(self, guild_ids: list[int]): - return [gid for gid in guild_ids if gid in self.bot.guild_ids] + return data From a2a27e9f2254d4c69c77ec9e0be52183c246b9e1 Mon Sep 17 00:00:00 2001 From: skelmis Date: Thu, 4 Jan 2024 18:03:28 +1300 Subject: [PATCH 03/22] fix: misc --- requirements.txt | 1 + suggestions/main.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 28afb05..848db35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,3 +43,4 @@ typing_extensions==4.3.0 websockets==10.4 yarl==1.7.2 zonis==1.2.5 +httpx diff --git a/suggestions/main.py b/suggestions/main.py index f314eb4..294b1a6 100644 --- a/suggestions/main.py +++ b/suggestions/main.py @@ -12,6 +12,7 @@ import aiohttp import cooldowns import disnake +import httpx from disnake import Locale from disnake.ext import commands from bot_base.paginators.disnake_paginator import DisnakePaginator @@ -27,9 +28,8 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot: is_prod: bool = True if os.environ.get("PROD", None) else False if is_prod: - async with aiohttp.ClientSession() as session: - async with session.get("http://localhost:7878/shard-count") as resp: - total_shards = int(await resp.text()) + request = httpx.get("http://localhost:7878/shard-count") + total_shards = int(request.text) cluster_id = int(os.environ["CLUSTER"]) offset = cluster_id - 1 number_of_shards_per_cluster = 10 From cc4fcee85b53ff2d8e6b0ea31350f8db34cd5dc6 Mon Sep 17 00:00:00 2001 From: skelmis Date: Thu, 4 Jan 2024 18:05:28 +1300 Subject: [PATCH 04/22] fix: misc --- suggestions/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/suggestions/main.py b/suggestions/main.py index 294b1a6..976b637 100644 --- a/suggestions/main.py +++ b/suggestions/main.py @@ -28,8 +28,9 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot: is_prod: bool = True if os.environ.get("PROD", None) else False if is_prod: - request = httpx.get("http://localhost:7878/shard-count") - total_shards = int(request.text) + # request = httpx.get("http://localhost:7878/shard-count") + # total_shards = int(request.text) + total_shards = 70 cluster_id = int(os.environ["CLUSTER"]) offset = cluster_id - 1 number_of_shards_per_cluster = 10 From 061e111f8f06d6c0c194bc8360d8c509b895cf2c Mon Sep 17 00:00:00 2001 From: skelmis Date: Thu, 4 Jan 2024 18:26:15 +1300 Subject: [PATCH 05/22] chore: move to 5 shards per cluster --- suggestions/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/suggestions/main.py b/suggestions/main.py index 976b637..0998a43 100644 --- a/suggestions/main.py +++ b/suggestions/main.py @@ -28,12 +28,13 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot: is_prod: bool = True if os.environ.get("PROD", None) else False if is_prod: + # TODO Fix this # request = httpx.get("http://localhost:7878/shard-count") # total_shards = int(request.text) total_shards = 70 cluster_id = int(os.environ["CLUSTER"]) offset = cluster_id - 1 - number_of_shards_per_cluster = 10 + number_of_shards_per_cluster = 5 shard_ids = [ i for i in range( From df5984f436d83864df0c28a91d7bd6504c94d908 Mon Sep 17 00:00:00 2001 From: skelmis Date: Thu, 4 Jan 2024 18:46:16 +1300 Subject: [PATCH 06/22] fix: application command sync ratelimits --- suggestions/bot.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/suggestions/bot.py b/suggestions/bot.py index 329f260..09cd438 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -143,10 +143,6 @@ async def dispatch_initial_ready(self): def total_cluster_count(self) -> int: return math.ceil(self.total_shards / 10) - @property - def is_primary_cluster(self) -> bool: - return bool(os.environ.get("IS_PRIMARY_CLUSTER", False)) - def error_embed( self, title: str, @@ -701,6 +697,24 @@ async def process_update_bot_listings(): state.add_background_task(task_1) log.info("Setup bot list updates") + @property + def is_primary_cluster(self) -> bool: + if not self.is_prod: + # Non-prod is always single cluster + return True + + shard_id = self.get_shard_id(self.main_guild_id) + return shard_id in self.shard_ids + + async def _sync_application_commands(self) -> None: + # In order to reduce getting rate-limited because every cluster + # decided it wants to sync application commands when it aint required + if not self.is_primary_cluster: + log.warning("Not syncing application commands as not primary cluster") + return + + await super()._sync_application_commands() + def get_shard_id(self, guild_id: Optional[int]) -> int: # DM's go to shard 0 shard_id = 0 From a923fdf7270f0bc0435eb615d6fb7cbb6bbdfd23 Mon Sep 17 00:00:00 2001 From: skelmis Date: Fri, 5 Jan 2024 22:03:45 +1300 Subject: [PATCH 07/22] feat: make total shards an env var --- suggestions/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suggestions/main.py b/suggestions/main.py index 0998a43..052f590 100644 --- a/suggestions/main.py +++ b/suggestions/main.py @@ -31,7 +31,7 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot: # TODO Fix this # request = httpx.get("http://localhost:7878/shard-count") # total_shards = int(request.text) - total_shards = 70 + total_shards = int(os.environ["TOTAL_SHARDS"]) cluster_id = int(os.environ["CLUSTER"]) offset = cluster_id - 1 number_of_shards_per_cluster = 5 From 01c054cdd5440d6a3d4568b99998be81207f40b0 Mon Sep 17 00:00:00 2001 From: skelmis Date: Sat, 6 Jan 2024 04:00:39 +1300 Subject: [PATCH 08/22] chore: debug RESUME's breaking --- requirements.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 848db35..8c2450d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,20 +2,26 @@ aiodns==3.0.0 aiohttp==3.8.1 aiosignal==1.2.0 alaric==1.2.0 +anyio==4.2.0 async-timeout==4.0.2 attrs==21.4.0 Bot-Base==1.7.1 Brotli==1.0.9 causar==0.2.0 cchardet==2.1.7 +certifi==2023.11.17 cffi==1.15.0 charset-normalizer==2.0.12 -disnake @ git+https://github.com/suggestionsbot/disnake.git@e489a5cd5561269a41a76b5037a38993886d7bfd +disnake @ git+https://github.com/suggestionsbot/disnake.git@7ee7822b6934117ba1c45669f548de663dd0f0b1 disnake-ext-components @ git+https://github.com/suggestionsbot/disnake-ext-components.git@91689ed74ffee73f631453a39e548af9b824826d dnspython==2.2.1 +exceptiongroup==1.2.0 frozenlist==1.3.0 function-cooldowns==1.3.1 graphviz==0.20.1 +h11==0.14.0 +httpcore==1.0.2 +httpx==0.26.0 humanize==4.2.0 idna==3.3 iniconfig==1.1.1 @@ -38,9 +44,9 @@ pytest==7.1.3 pytest-asyncio==0.19.0 python-dotenv==0.20.0 sentinels==1.0.0 +sniffio==1.3.0 tomli==2.0.1 typing_extensions==4.3.0 websockets==10.4 yarl==1.7.2 zonis==1.2.5 -httpx From a61ab1b19fb1536ec0efed1a2d939147327f2dcc Mon Sep 17 00:00:00 2001 From: skelmis Date: Sat, 6 Jan 2024 17:13:48 +1300 Subject: [PATCH 09/22] chore: debug RESUME's breaking --- main.py | 22 +++++++++++----------- requirements.txt | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/main.py b/main.py index bfbe81b..d18e928 100644 --- a/main.py +++ b/main.py @@ -12,22 +12,22 @@ load_dotenv() logging.basicConfig( - level=logging.INFO, + level=logging.DEBUG, format="%(levelname)-8s | %(asctime)s | %(filename)19s:%(funcName)-27s | %(message)s", datefmt="%d/%m/%Y %I:%M:%S %p", ) # logging.getLogger("asyncio").setLevel(logging.DEBUG) -disnake_logger = logging.getLogger("disnake") -disnake_logger.setLevel(logging.INFO) -gateway_logger = logging.getLogger("disnake.gateway") -gateway_logger.setLevel(logging.WARNING) -client_logger = logging.getLogger("disnake.client") -client_logger.setLevel(logging.WARNING) -http_logger = logging.getLogger("disnake.http") -http_logger.setLevel(logging.WARNING) -shard_logger = logging.getLogger("disnake.shard") -shard_logger.setLevel(logging.WARNING) +# disnake_logger = logging.getLogger("disnake") +# disnake_logger.setLevel(logging.INFO) +# gateway_logger = logging.getLogger("disnake.gateway") +# gateway_logger.setLevel(logging.WARNING) +# client_logger = logging.getLogger("disnake.client") +# client_logger.setLevel(logging.WARNING) +# http_logger = logging.getLogger("disnake.http") +# http_logger.setLevel(logging.WARNING) +# shard_logger = logging.getLogger("disnake.shard") +# shard_logger.setLevel(logging.WARNING) suggestions_logger = logging.getLogger("suggestions") suggestions_logger.setLevel(logging.DEBUG) diff --git a/requirements.txt b/requirements.txt index 8c2450d..a3ff0d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ cchardet==2.1.7 certifi==2023.11.17 cffi==1.15.0 charset-normalizer==2.0.12 -disnake @ git+https://github.com/suggestionsbot/disnake.git@7ee7822b6934117ba1c45669f548de663dd0f0b1 +disnake @ git+https://github.com/suggestionsbot/disnake.git@22a572afd139144c0e2de49c7692a73ab74d8a3d disnake-ext-components @ git+https://github.com/suggestionsbot/disnake-ext-components.git@91689ed74ffee73f631453a39e548af9b824826d dnspython==2.2.1 exceptiongroup==1.2.0 From e69fd847116062a269714c97ac4ce54605616492 Mon Sep 17 00:00:00 2001 From: skelmis Date: Sat, 6 Jan 2024 17:16:51 +1300 Subject: [PATCH 10/22] chore: debug RESUME's breaking --- suggestions/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/suggestions/main.py b/suggestions/main.py index 052f590..12217bc 100644 --- a/suggestions/main.py +++ b/suggestions/main.py @@ -34,7 +34,7 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot: total_shards = int(os.environ["TOTAL_SHARDS"]) cluster_id = int(os.environ["CLUSTER"]) offset = cluster_id - 1 - number_of_shards_per_cluster = 5 + number_of_shards_per_cluster = int(os.environ["SHARDS_PER_CLUSTER"]) shard_ids = [ i for i in range( From 2a8ceada08f60ff737bf808981e39c133e0b4f8a Mon Sep 17 00:00:00 2001 From: skelmis Date: Sat, 6 Jan 2024 17:30:56 +1300 Subject: [PATCH 11/22] chore: debug bot breaking --- suggestions/bot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/suggestions/bot.py b/suggestions/bot.py index 09cd438..d1e1c70 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -16,7 +16,7 @@ from alaric import Cursor from bot_base.wraps import WrappedChannel from cooldowns import CallableOnCooldown -from disnake import Locale, LocalizationKeyError +from disnake import Locale, LocalizationKeyError, GatewayParams from disnake.ext import commands from bot_base import BotBase, BotContext, PrefixNotFound @@ -103,6 +103,7 @@ def __init__(self, *args, **kwargs): name="suggestions", type=disnake.ActivityType.watching, ), + gateway_params=GatewayParams(zlib=False), ) self._has_dispatched_initial_ready: bool = False From c5727a6658c8f2dd5f1daf631fb9cd7e2a75bf1f Mon Sep 17 00:00:00 2001 From: skelmis Date: Sat, 6 Jan 2024 17:50:49 +1300 Subject: [PATCH 12/22] chore: revert to non GW --- main.py | 20 ++++++++++---------- suggestions/bot.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index d18e928..435e926 100644 --- a/main.py +++ b/main.py @@ -18,16 +18,16 @@ ) # logging.getLogger("asyncio").setLevel(logging.DEBUG) -# disnake_logger = logging.getLogger("disnake") -# disnake_logger.setLevel(logging.INFO) -# gateway_logger = logging.getLogger("disnake.gateway") -# gateway_logger.setLevel(logging.WARNING) -# client_logger = logging.getLogger("disnake.client") -# client_logger.setLevel(logging.WARNING) -# http_logger = logging.getLogger("disnake.http") -# http_logger.setLevel(logging.WARNING) -# shard_logger = logging.getLogger("disnake.shard") -# shard_logger.setLevel(logging.WARNING) +disnake_logger = logging.getLogger("disnake") +disnake_logger.setLevel(logging.INFO) +gateway_logger = logging.getLogger("disnake.gateway") +gateway_logger.setLevel(logging.WARNING) +client_logger = logging.getLogger("disnake.client") +client_logger.setLevel(logging.WARNING) +http_logger = logging.getLogger("disnake.http") +http_logger.setLevel(logging.WARNING) +shard_logger = logging.getLogger("disnake.shard") +shard_logger.setLevel(logging.WARNING) suggestions_logger = logging.getLogger("suggestions") suggestions_logger.setLevel(logging.DEBUG) diff --git a/suggestions/bot.py b/suggestions/bot.py index d1e1c70..d7b14ab 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -103,7 +103,7 @@ def __init__(self, *args, **kwargs): name="suggestions", type=disnake.ActivityType.watching, ), - gateway_params=GatewayParams(zlib=False), + # gateway_params=GatewayParams(zlib=False), ) self._has_dispatched_initial_ready: bool = False From 60711592f719c2a70eaa28681df5df68c4587626 Mon Sep 17 00:00:00 2001 From: skelmis Date: Sat, 6 Jan 2024 17:58:15 +1300 Subject: [PATCH 13/22] chore: revert to non GW --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 435e926..bfbe81b 100644 --- a/main.py +++ b/main.py @@ -12,7 +12,7 @@ load_dotenv() logging.basicConfig( - level=logging.DEBUG, + level=logging.INFO, format="%(levelname)-8s | %(asctime)s | %(filename)19s:%(funcName)-27s | %(message)s", datefmt="%d/%m/%Y %I:%M:%S %p", ) From 0195bd125f060b50dd8cf7bbcca82edb420f69f9 Mon Sep 17 00:00:00 2001 From: Ethan <47520067+Skelmis@users.noreply.github.com> Date: Sat, 6 Jan 2024 19:29:02 +1300 Subject: [PATCH 14/22] chore: refactor garven (#72) * feat: begin migration * feat: move cluster status * chore: fix imports --- requirements.txt | 7 ----- suggestions/bot.py | 35 ++++++++--------------- suggestions/cogs/help_guild_cog.py | 13 +-------- suggestions/exceptions.py | 4 +++ suggestions/garven.py | 46 ++++++++++++++++++++++++++++++ suggestions/main.py | 5 ---- suggestions/zonis_routes.py | 7 +---- 7 files changed, 64 insertions(+), 53 deletions(-) diff --git a/requirements.txt b/requirements.txt index a3ff0d5..558ece6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,26 +2,20 @@ aiodns==3.0.0 aiohttp==3.8.1 aiosignal==1.2.0 alaric==1.2.0 -anyio==4.2.0 async-timeout==4.0.2 attrs==21.4.0 Bot-Base==1.7.1 Brotli==1.0.9 causar==0.2.0 cchardet==2.1.7 -certifi==2023.11.17 cffi==1.15.0 charset-normalizer==2.0.12 disnake @ git+https://github.com/suggestionsbot/disnake.git@22a572afd139144c0e2de49c7692a73ab74d8a3d disnake-ext-components @ git+https://github.com/suggestionsbot/disnake-ext-components.git@91689ed74ffee73f631453a39e548af9b824826d dnspython==2.2.1 -exceptiongroup==1.2.0 frozenlist==1.3.0 function-cooldowns==1.3.1 graphviz==0.20.1 -h11==0.14.0 -httpcore==1.0.2 -httpx==0.26.0 humanize==4.2.0 idna==3.3 iniconfig==1.1.1 @@ -44,7 +38,6 @@ pytest==7.1.3 pytest-asyncio==0.19.0 python-dotenv==0.20.0 sentinels==1.0.0 -sniffio==1.3.0 tomli==2.0.1 typing_extensions==4.3.0 websockets==10.4 diff --git a/suggestions/bot.py b/suggestions/bot.py index d7b14ab..4343e53 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -34,6 +34,7 @@ UnhandledError, QueueImbalance, BlocklistedUser, + PartialResponse, ) from suggestions.http_error_parser import try_parse_http_error from suggestions.objects import Error, GuildConfig, UserConfig @@ -652,25 +653,9 @@ async def process_update_bot_listings(): headers = {"Authorization": os.environ["SUGGESTIONS_API_KEY"]} while not state.is_closing: - url = ( - "https://garven.suggestions.gg/aggregate/guilds/count" - if self.is_prod - else "https://garven.dev.suggestions.gg/aggregate/guilds/count" - ) - async with aiohttp.ClientSession( - headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]} - ) as session: - async with session.get(url) as resp: - data: dict = await resp.json() - if resp.status != 200: - log.error("Stopping bot list updates") - log.error("%s", data) - break - - if data["partial_response"]: - log.warning( - "Skipping bot list updates as IPC returned a partial responses" - ) + try: + total_guilds = await self.garven.get_total_guilds() + except PartialResponse: await self.sleep_with_condition( time_between_updates.total_seconds(), lambda: self.state.is_closing, @@ -678,12 +663,15 @@ async def process_update_bot_listings(): continue body = { - "guild_count": int(data["statistic"]), + "guild_count": int(total_guilds), "shard_count": int(self.shard_count), } async with aiohttp.ClientSession(headers=headers) as session: async with session.post( - os.environ["SUGGESTIONS_STATS_API_URL"], json=body + os.environ[ + "SUGGESTIONS_STATS_API_URL" + ], # This is the bot list API # lists.suggestions.gg + json=body, ) as r: if r.status != 200: log.warning("%s", r.text) @@ -834,13 +822,14 @@ async def inner(): log.error("Borked it") return + tb = "".join(traceback.format_exception(e)) log.error( "Status update failed: %s", - "".join(traceback.format_exception(e)), + tb, ) await self.garven.notify_devs( title="Status page ping error", - description=str(e), + description=tb, sender=f"Cluster {self.cluster_id}, shard {self.shard_id}", ) diff --git a/suggestions/cogs/help_guild_cog.py b/suggestions/cogs/help_guild_cog.py index 4b1d602..e2b218c 100644 --- a/suggestions/cogs/help_guild_cog.py +++ b/suggestions/cogs/help_guild_cog.py @@ -143,11 +143,6 @@ async def show_bot_status( red_circle = "🔴" green_circle = "🟢" - url = ( - "https://garven.suggestions.gg/cluster/status" - if self.bot.is_prod - else "https://garven.dev.suggestions.gg/cluster/status" - ) embed = disnake.Embed( timestamp=datetime.datetime.utcnow(), @@ -156,13 +151,7 @@ async def show_bot_status( down_shards: list[str] = [str(i) for i in range(53)] down_clusters: list[str] = [str(i) for i in range(1, 7)] avg_bot_latency: list[float] = [] - async with aiohttp.ClientSession( - headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]} - ) as session: - async with session.get(url) as resp: - data: dict[str, dict | bool] = await resp.json() - if resp.status != 200: - log.error("Something went wrong: %s", data) + data = await self.bot.garven.cluster_status() if data.pop("partial_response") is not None: embed.set_footer(text="Partial response") diff --git a/suggestions/exceptions.py b/suggestions/exceptions.py index 1e78006..6669a63 100644 --- a/suggestions/exceptions.py +++ b/suggestions/exceptions.py @@ -49,3 +49,7 @@ class QueueImbalance(disnake.DiscordException): class BlocklistedUser(CheckFailure): """This user is blocked from taking this action in this guild.""" + + +class PartialResponse(Exception): + """A garven route returned a partial response when we require a full response""" diff --git a/suggestions/garven.py b/suggestions/garven.py index a2891c4..2dabf32 100644 --- a/suggestions/garven.py +++ b/suggestions/garven.py @@ -6,6 +6,8 @@ import aiohttp +from suggestions.exceptions import PartialResponse + if TYPE_CHECKING: from suggestions import SuggestionsBot @@ -19,12 +21,30 @@ def __init__(self, bot: SuggestionsBot): if bot.is_prod else "https://garven.dev.suggestions.gg" ) + self._ws_url = ( + "wss://garven.suggestions.gg/ws" + if bot.is_prod + else "wss://garven.dev.suggestions.gg/ws" + ) self._session: aiohttp.ClientSession = aiohttp.ClientSession( base_url=self._url, headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]}, ) self.bot: SuggestionsBot = bot + @property + def http_url(self) -> str: + return self._url + + @property + def ws_url(self) -> str: + return self._ws_url + + @staticmethod + async def _handle_status(resp: aiohttp.ClientResponse): + if resp.status > 299: + raise ValueError(f"Garven route failed {resp.url}") + async def notify_devs(self, *, title: str, description: str, sender: str): async with self._session.post( "/cluster/notify_devs", @@ -39,3 +59,29 @@ async def notify_devs(self, *, title: str, description: str, sender: str): "Error when attempting to notify devs\n\t- %s", await resp.text(), ) + + async def get_shard_info(self, guild_id: int) -> dict[str, str]: + async with self._session.get( + f"/aggregate/guilds/{guild_id}/shard_info" + ) as resp: + await self._handle_status(resp) + data = await resp.json() + + return data + + async def get_total_guilds(self) -> int: + async with self._session.get("/aggregate/guilds/count") as resp: + await self._handle_status(resp) + data = await resp.json() + if data["partial_response"]: + log.warning("get_total_guilds returned a partial response") + raise PartialResponse + + return data["statistic"] + + async def cluster_status(self) -> dict: + async with self._session.get("/cluster/status") as resp: + await self._handle_status(resp) + data = await resp.json() + + return data diff --git a/suggestions/main.py b/suggestions/main.py index 12217bc..0f8c30c 100644 --- a/suggestions/main.py +++ b/suggestions/main.py @@ -9,10 +9,8 @@ import textwrap from traceback import format_exception -import aiohttp import cooldowns import disnake -import httpx from disnake import Locale from disnake.ext import commands from bot_base.paginators.disnake_paginator import DisnakePaginator @@ -28,9 +26,6 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot: is_prod: bool = True if os.environ.get("PROD", None) else False if is_prod: - # TODO Fix this - # request = httpx.get("http://localhost:7878/shard-count") - # total_shards = int(request.text) total_shards = int(os.environ["TOTAL_SHARDS"]) cluster_id = int(os.environ["CLUSTER"]) offset = cluster_id - 1 diff --git a/suggestions/zonis_routes.py b/suggestions/zonis_routes.py index 5e9b284..0e7c359 100644 --- a/suggestions/zonis_routes.py +++ b/suggestions/zonis_routes.py @@ -19,13 +19,8 @@ class ZonisRoutes: def __init__(self, bot: SuggestionsBot): self.bot: SuggestionsBot = bot - url = ( - "wss://garven.suggestions.gg/ws" - if self.bot.is_prod - else "wss://garven.dev.suggestions.gg/ws" - ) self.client: client.Client = client.Client( - url=url, + url=bot.garven.ws_url, identifier=str(bot.cluster_id), secret_key=os.environ["ZONIS_SECRET_KEY"], override_key=os.environ.get("ZONIS_OVERRIDE_KEY"), From d1bb2c5a9293e381fa9f78ea31c20b5345682a72 Mon Sep 17 00:00:00 2001 From: skelmis Date: Thu, 11 Jan 2024 23:03:04 +1300 Subject: [PATCH 15/22] fix: exceptions as a result of an edit race condition --- suggestions/clunk2/edits.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/suggestions/clunk2/edits.py b/suggestions/clunk2/edits.py index 9014983..9149c21 100644 --- a/suggestions/clunk2/edits.py +++ b/suggestions/clunk2/edits.py @@ -29,6 +29,16 @@ async def update_suggestion_message( pending_edits.add(suggestion.suggestion_id) await asyncio.sleep(time_after) + if suggestion.channel_id is None or suggestion.message_id is None: + log.debug( + "Suggestion %s had a NoneType by the time it was to be edited channel_id=%s, message_id=%s", + suggestion.suggestion_id, + suggestion.channel_id, + suggestion.message_id, + ) + pending_edits.discard(suggestion.suggestion_id) + return + try: await MessageEditing( bot, channel_id=suggestion.channel_id, message_id=suggestion.message_id From 025c9e193f05afa9f9c4e1282d2ca8e4c476969e Mon Sep 17 00:00:00 2001 From: skelmis Date: Thu, 11 Jan 2024 23:54:27 +1300 Subject: [PATCH 16/22] feat: add more debugging routes --- suggestions/bot.py | 57 +++++++++++++++++++++++++++++++++++++ suggestions/garven.py | 9 ++++++ suggestions/zonis_routes.py | 22 +++++++++++++- 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/suggestions/bot.py b/suggestions/bot.py index 4343e53..ebbcab5 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -2,6 +2,7 @@ import asyncio import datetime +import io import logging import math import os @@ -571,6 +572,7 @@ async def load(self): await self.stats.load() await self.update_bot_listings() await self.push_status() + await self.update_dev_channel() await self.watch_for_shutdown_request() await self.load_cogs() await self.zonis.start() @@ -635,6 +637,61 @@ async def process_watch_for_shutdown(): process_watch_for_shutdown.__task = task_1 state.add_background_task(task_1) + async def update_dev_channel(self): + if not self.is_prod: + log.info("Not watching for debug info as not on prod") + return + + state: State = self.state + + async def process_watch_for_shutdown(): + await self.wait_until_ready() + log.debug("Started tracking bot latency") + + while not state.is_closing: + # Update once an hour + await self.sleep_with_condition( + datetime.timedelta(minutes=5).total_seconds(), + lambda: self.state.is_closing, + ) + + await self.garven.notify_devs( + title=f"WS latency as follows", + description=f"Timestamped for {datetime.datetime.utcnow().isoformat()}", + sender=f"N/A", + ) + + data = await self.garven.get_bot_ws_latency() + shard_data = data["shards"] + for i in range(0, 75, 5): + description = io.StringIO() + for o in range(0, 6): + shard = str(i + o) + try: + description.write( + f"**Shard {shard}**\nWS latency: `{shard_data[shard]['ws']}`\n" + f"Keep Alive latency: `{shard_data[shard]['keepalive']}`\n\n" + ) + except KeyError: + # My lazy way of not doing env checks n math right + continue + + if description.getvalue(): + await self.garven.notify_devs( + title=f"WS latency", + description=description.getvalue(), + sender=f"Partial response: {data['partial_response']}", + ) + + await self.sleep_with_condition( + datetime.timedelta(hours=1).total_seconds(), + lambda: self.state.is_closing, + ) + + task_1 = asyncio.create_task(process_watch_for_shutdown()) + process_watch_for_shutdown.__task = task_1 + state.add_background_task(task_1) + async def update_bot_listings(self) -> None: """Updates the bot lists with current stats.""" if not self.is_prod: diff --git a/suggestions/garven.py b/suggestions/garven.py index 2dabf32..bb04931 100644 --- a/suggestions/garven.py +++ b/suggestions/garven.py @@ -21,11 +21,13 @@ def __init__(self, bot: SuggestionsBot): if bot.is_prod else "https://garven.dev.suggestions.gg" ) + # self._url = "http://127.0.0.1:8002" self._ws_url = ( "wss://garven.suggestions.gg/ws" if bot.is_prod else "wss://garven.dev.suggestions.gg/ws" ) + # self._ws_url = "ws://127.0.0.1:8002/ws" self._session: aiohttp.ClientSession = aiohttp.ClientSession( base_url=self._url, headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]}, @@ -85,3 +87,10 @@ async def cluster_status(self) -> dict: data = await resp.json() return data + + async def get_bot_ws_latency(self): + async with self._session.get("/cluster/latency/ws") as resp: + await self._handle_status(resp) + data = await resp.json() + + return data diff --git a/suggestions/zonis_routes.py b/suggestions/zonis_routes.py index 0e7c359..4ca947b 100644 --- a/suggestions/zonis_routes.py +++ b/suggestions/zonis_routes.py @@ -3,7 +3,7 @@ import logging import math import os -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal import disnake from zonis import client @@ -33,6 +33,7 @@ def __init__(self, bot: SuggestionsBot): "refresh_premium", "shared_guilds", "cached_item_count", + "cluster_ws_status", ) async def start(self): @@ -65,6 +66,25 @@ async def cluster_status(self): return data + @client.route() + async def cluster_ws_status( + self, + ) -> dict[str, dict[Literal["ws", "keepalive"], str]]: + data: dict[str, dict[Literal["ws", "keepalive"], str]] = {} + for shard_id, shard_info in self.bot.shards.items(): + shard_data: dict[Literal["ws", "keepalive"], str] = { + "ws": str(round(shard_info.latency, 5)) + } + wsc = shard_info._parent.ws._keep_alive + if wsc is None: + shard_data["keepalive"] = "None" + else: + shard_data["keepalive"] = str(round(wsc.latency, 5)) + + data[str(shard_id)] = shard_data + + return data + @client.route() async def share_with_devs(self, title, description, sender): channel: disnake.TextChannel = await self.bot.get_or_fetch_channel( # type: ignore From bb9f44b7812af625e02a908378bcdfb2062b1626 Mon Sep 17 00:00:00 2001 From: skelmis Date: Thu, 11 Jan 2024 23:55:40 +1300 Subject: [PATCH 17/22] fix: WS debug check --- suggestions/bot.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/suggestions/bot.py b/suggestions/bot.py index ebbcab5..3652557 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -642,6 +642,10 @@ async def update_dev_channel(self): log.info("Not watching for debug info as not on prod") return + if not self.is_primary_cluster: + log.info("Not watching for debug info as not primary cluster") + return + state: State = self.state async def process_watch_for_shutdown(): From ba699cc95b95fe0a13c1c66ea2b269a55c42b068 Mon Sep 17 00:00:00 2001 From: skelmis Date: Thu, 18 Jan 2024 21:26:22 +1300 Subject: [PATCH 18/22] chore: add gc.collect during on_resumed #44 --- suggestions/bot.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/suggestions/bot.py b/suggestions/bot.py index 3652557..7cec9fe 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -2,6 +2,7 @@ import asyncio import datetime +import gc import io import logging import math @@ -91,6 +92,7 @@ def __init__(self, *args, **kwargs): "vote", } self.converted_prefix_commands: set[str] = {"suggest", "approve", "reject"} + self.gc_lock: asyncio.Lock = asyncio.Lock() # Sharding info self.cluster_id: int = kwargs.pop("cluster", 0) @@ -142,6 +144,15 @@ async def dispatch_initial_ready(self): log.info("Startup took: %s", self.get_uptime()) await self.suggestion_emojis.populate_emojis() + async def on_resumed(self): + if self.gc_lock.locked(): + return + + async with self.gc_lock: + await asyncio.sleep(2.0) + collected = gc.collect() + log.info(f"Garbage collector: collected {collected} objects.") + @property def total_cluster_count(self) -> int: return math.ceil(self.total_shards / 10) From 806ba53c6a7bb8b63225d19a0f8e6f5ce3909f4c Mon Sep 17 00:00:00 2001 From: Ethan <47520067+Skelmis@users.noreply.github.com> Date: Sun, 4 Feb 2024 21:58:29 +1300 Subject: [PATCH 19/22] chore: add TTL from last access to paginators (#74) * chore: add TTL from last access to paginators * chore: migrate all TimedCache usages to python commons * feat: add ttl from last access to a few caches --- requirements.txt | 8 ++ suggestions/bot.py | 5 +- suggestions/clunk/__init__.py | 3 - suggestions/clunk/cache.py | 25 ----- suggestions/clunk/clunk.py | 32 ------ suggestions/clunk/lock.py | 124 ----------------------- suggestions/cogs/suggestion_queue_cog.py | 7 +- suggestions/state.py | 19 ++-- suggestions/stats.py | 6 +- tests/conftest.py | 12 --- tests/test_clunk.py | 93 ----------------- 11 files changed, 28 insertions(+), 306 deletions(-) delete mode 100644 suggestions/clunk/__init__.py delete mode 100644 suggestions/clunk/cache.py delete mode 100644 suggestions/clunk/clunk.py delete mode 100644 suggestions/clunk/lock.py delete mode 100644 tests/test_clunk.py diff --git a/requirements.txt b/requirements.txt index 558ece6..b64820a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,20 +2,26 @@ aiodns==3.0.0 aiohttp==3.8.1 aiosignal==1.2.0 alaric==1.2.0 +anyio==4.2.0 async-timeout==4.0.2 attrs==21.4.0 Bot-Base==1.7.1 Brotli==1.0.9 causar==0.2.0 cchardet==2.1.7 +certifi==2023.11.17 cffi==1.15.0 charset-normalizer==2.0.12 disnake @ git+https://github.com/suggestionsbot/disnake.git@22a572afd139144c0e2de49c7692a73ab74d8a3d disnake-ext-components @ git+https://github.com/suggestionsbot/disnake-ext-components.git@91689ed74ffee73f631453a39e548af9b824826d dnspython==2.2.1 +exceptiongroup==1.2.0 frozenlist==1.3.0 function-cooldowns==1.3.1 graphviz==0.20.1 +h11==0.14.0 +httpcore==1.0.2 +httpx==0.26.0 humanize==4.2.0 idna==3.3 iniconfig==1.1.1 @@ -38,6 +44,8 @@ pytest==7.1.3 pytest-asyncio==0.19.0 python-dotenv==0.20.0 sentinels==1.0.0 +skelmis-commons==1.1.0 +sniffio==1.3.0 tomli==2.0.1 typing_extensions==4.3.0 websockets==10.4 diff --git a/suggestions/bot.py b/suggestions/bot.py index 7cec9fe..594ffb4 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -23,7 +23,6 @@ from bot_base import BotBase, BotContext, PrefixNotFound from suggestions import State, Colors, Emojis, ErrorCode, Garven -from suggestions.clunk import Clunk from suggestions.exceptions import ( BetaOnly, MissingSuggestionsChannel, @@ -49,7 +48,7 @@ class SuggestionsBot(commands.AutoShardedInteractionBot, BotBase): def __init__(self, *args, **kwargs): - self.version: str = "Public Release 3.20" + self.version: str = "Public Release 3.21" self.main_guild_id: int = 601219766258106399 self.legacy_beta_role_id: int = 995588041991274547 self.automated_beta_role_id: int = 998173237282361425 @@ -75,7 +74,6 @@ def __init__(self, *args, **kwargs): self.state: State = State(self.db, self) self.stats: Stats = Stats(self) self.garven: Garven = Garven(self) - self.clunk: Clunk = Clunk(self.state) self.suggestion_emojis: Emojis = Emojis(self) self.old_prefixed_commands: set[str] = { "changelog", @@ -595,7 +593,6 @@ async def graceful_shutdown(self) -> None: """ log.debug("Attempting to shutdown") self.state.notify_shutdown() - await self.clunk.kill_all() await self.zonis.client.close() await asyncio.gather(*self.state.background_tasks) log.info("Shutting down") diff --git a/suggestions/clunk/__init__.py b/suggestions/clunk/__init__.py deleted file mode 100644 index 6bbec54..0000000 --- a/suggestions/clunk/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .lock import ClunkLock -from .cache import ClunkCache -from .clunk import Clunk diff --git a/suggestions/clunk/cache.py b/suggestions/clunk/cache.py deleted file mode 100644 index 7d2258a..0000000 --- a/suggestions/clunk/cache.py +++ /dev/null @@ -1,25 +0,0 @@ -from bot_base.caches import TimedCache - - -class ClunkCache(TimedCache): - def __contains__(self, item): - try: - entry = self.cache[item] - if not entry.value.has_requests: - entry.value.kill() - self.cache.pop(item) - return False - except KeyError: - return False - else: - return True - - def force_clean(self) -> None: - items = {} - for k, v in self.cache.items(): - if v.value.has_requests: - items[k] = v - else: - v.value.kill() - - self.cache = items diff --git a/suggestions/clunk/clunk.py b/suggestions/clunk/clunk.py deleted file mode 100644 index cdfb6da..0000000 --- a/suggestions/clunk/clunk.py +++ /dev/null @@ -1,32 +0,0 @@ -import asyncio -import logging -from datetime import timedelta - -from bot_base import NonExistentEntry - -from suggestions import State -from suggestions.clunk import ClunkCache, ClunkLock - -log = logging.getLogger(__name__) - - -class Clunk: - def __init__(self, state: State): - self._state: State = state - self._cache: ClunkCache[str, ClunkLock] = ClunkCache( - lazy_eviction=False, global_ttl=timedelta(hours=1) - ) - - def acquire(self, suggestion_id: str) -> ClunkLock: - key = suggestion_id - try: - return self._cache.get_entry(key) - except NonExistentEntry: - lock = ClunkLock(self._state) - self._cache.add_entry(key, lock) - return lock - - async def kill_all(self) -> None: - """Kill all current ClunkLock instances.""" - for lock in self._cache.cache.values(): - lock.value.kill() diff --git a/suggestions/clunk/lock.py b/suggestions/clunk/lock.py deleted file mode 100644 index 5886b18..0000000 --- a/suggestions/clunk/lock.py +++ /dev/null @@ -1,124 +0,0 @@ -import asyncio -from typing import Coroutine - -from suggestions import State - - -class ClunkLock: - """Custom request processing for the 21st century. - - Example: - - We need to edit a message 9 times. - Each time the message is edited we simply increment a counter. - - For bulk operations this causes rate-limits to be hit - as we process the edits one by one until we exhaust the requests. - - Given a FIFO queue however, we can in theory skip requests - to reduce the amount of edits required to reach the same result. - - For example, the below two approaches would result in the same - output for the end user but the proposed design would only - make two requests instead of the nine requests it currently takes - which should alleviate errors we run into due to rate-limits. - - Currently: - +---+---+---+---+---+---+---+---+---+ - | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | - +---+---+---+---+---+---+---+---+---+ - - Proposed: - +---+---+ - | 1 | 9 | - +---+---+ - - Find the generic version here: https://workbin.dev/?id=1664955297537152 - """ - - def __init__(self, state: State): - self._state: State = state - self.__killed: bool = False - self.is_currently_running: bool = False - self._next_request: Coroutine | None = None - self._event: asyncio.Event = asyncio.Event() - self._current_request: Coroutine | None = None - - @property - def has_requests(self) -> None: - return self._current_request is not None - - @property - def __is_closing(self) -> bool: - return self._state.is_closing or self.__killed - - async def wait(self) -> None: - """Block until all requests are processed.""" - while self.has_requests: - await asyncio.sleep(0) - - def kill(self) -> None: - """Kill any current requests inline with a graceful shutdown.""" - self.__killed = True - if not self._event.is_set(): - self._event.set() - - async def run(self) -> None: - """Begin processing the queue in the background. - - Notes - ----- - The program lifetime must be longer then the - lifetime of the tasks enqueued otherwise tasks - may fail to be awaited before the program dies. - """ - if self.is_currently_running: - return - - asyncio.create_task(self._run()) - self.is_currently_running = True - - async def _run(self) -> None: - while not self.__is_closing: - await self._event.wait() - if self.__is_closing: - break - - await self._current_request - - if self._next_request: - self._current_request = self._next_request - self._next_request = None - else: - self._current_request = None - self._event.clear() - - self.is_currently_running = False - # Cancel any remaining coros to supress - # warnings since we want to shut this down - if self._current_request: - asyncio.create_task(self._current_request).cancel() - if self._next_request: - asyncio.create_task(self._next_request).cancel() - - def enqueue(self, request: Coroutine) -> None: - """Add a request to the queue for processing - - Parameters - ---------- - request: Coroutine - The request to queue for processing - """ - if self._current_request: - # We are already processing a request - # so queue this for future running - if self._next_request: - # Cancel the old coroutine to supress warnings - # about 'func' was never awaited - asyncio.create_task(self._next_request).cancel() - self._next_request = request - else: - # We aren't processing any requests - # so queue this for immediate running - self._current_request = request - self._event.set() diff --git a/suggestions/cogs/suggestion_queue_cog.py b/suggestions/cogs/suggestion_queue_cog.py index 98b0594..5f29c86 100644 --- a/suggestions/cogs/suggestion_queue_cog.py +++ b/suggestions/cogs/suggestion_queue_cog.py @@ -10,8 +10,7 @@ from alaric.comparison import EQ from alaric.logical import AND from alaric.projections import Projection, SHOW -from bot_base import NonExistentEntry -from bot_base.caches import TimedCache +from commons.caching import NonExistentEntry, TimedCache from disnake import Guild from disnake.ext import commands, components @@ -35,7 +34,9 @@ def __init__(self, bot): self.state = self.bot.state self.queued_suggestions_db: Document = self.bot.db.queued_suggestions self.paginator_objects: TimedCache = TimedCache( - global_ttl=timedelta(minutes=15), lazy_eviction=False + global_ttl=timedelta(minutes=15), + lazy_eviction=False, + ttl_from_last_access=True, ) async def get_paginator_for( diff --git a/suggestions/state.py b/suggestions/state.py index befaa6a..6cf20ba 100644 --- a/suggestions/state.py +++ b/suggestions/state.py @@ -14,8 +14,7 @@ from alaric.logical import AND from alaric.meta import Negate from alaric.projections import PROJECTION, SHOW -from bot_base import NonExistentEntry -from bot_base.caches import TimedCache +from commons.caching import NonExistentEntry, TimedCache from suggestions.objects import GuildConfig, UserConfig @@ -42,17 +41,25 @@ def __init__(self, database: SuggestionsMongoManager, bot: SuggestionsBot): ) self.guild_cache_ttl: timedelta = timedelta(minutes=15) self.guild_cache: TimedCache[int, disnake.Guild] = TimedCache( - global_ttl=self.guild_cache_ttl, lazy_eviction=False + global_ttl=self.guild_cache_ttl, + lazy_eviction=False, + ttl_from_last_access=True, ) self.view_voters_cache: TimedCache[int, list[str]] = TimedCache( - global_ttl=self.autocomplete_cache_ttl, lazy_eviction=False + global_ttl=self.autocomplete_cache_ttl, + lazy_eviction=False, + ttl_from_last_access=True, ) self.guild_configs: TimedCache = TimedCache( - global_ttl=timedelta(minutes=30), lazy_eviction=False + global_ttl=timedelta(minutes=30), + lazy_eviction=False, + ttl_from_last_access=True, ) self.user_configs: TimedCache = TimedCache( - global_ttl=timedelta(minutes=30), lazy_eviction=False + global_ttl=timedelta(minutes=30), + lazy_eviction=False, + ttl_from_last_access=True, ) self.existing_error_ids: Set[str] = set() diff --git a/suggestions/stats.py b/suggestions/stats.py index c26e52f..28ab7be 100644 --- a/suggestions/stats.py +++ b/suggestions/stats.py @@ -4,14 +4,12 @@ import datetime import logging from enum import Enum -from typing import TYPE_CHECKING, Optional, Dict, Literal, Union, Type +from typing import TYPE_CHECKING, Optional, Type import alaric from alaric import Cursor, AQ from alaric.comparison import EQ -from alaric.types import ObjectId -from bot_base.caches import TimedCache -from motor.motor_asyncio import AsyncIOMotorCollection +from commons.caching import TimedCache from suggestions.objects.stats import MemberStats, MemberCommandStats diff --git a/tests/conftest.py b/tests/conftest.py index 24b15b6..d63f866 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,11 @@ import os from unittest.mock import AsyncMock -import disnake import pytest from causar import Causar, InjectionMetadata import suggestions -from suggestions.clunk import ClunkLock, Clunk from tests.mocks import MockedSuggestionsMongoManager @@ -43,13 +41,3 @@ async def injection_metadata(causar: Causar) -> InjectionMetadata: return InjectionMetadata( guild_id=881118111967883295, channel_id=causar.faker.generate_snowflake() ) - - -@pytest.fixture -async def clunk_lock(causar: Causar) -> ClunkLock: - return ClunkLock(causar.bot.state) # type: ignore - - -@pytest.fixture -async def clunk(causar: Causar) -> Clunk: - return Clunk(causar.bot.state) # type: ignore diff --git a/tests/test_clunk.py b/tests/test_clunk.py deleted file mode 100644 index a6d6625..0000000 --- a/tests/test_clunk.py +++ /dev/null @@ -1,93 +0,0 @@ -import asyncio - -from suggestions.clunk import Clunk, ClunkLock - - -async def test_pre_queue(clunk_lock: ClunkLock): - data = [] - - async def append(number): - data.append(number) - - clunk_lock.enqueue(append(1)) - clunk_lock.enqueue(append(2)) - clunk_lock.enqueue(append(3)) - - await clunk_lock.run() - await clunk_lock.wait() - - assert data == [1, 3] - - clunk_lock.kill() - - -async def test_post_queue_instant(clunk_lock: ClunkLock): - data = [] - - async def append(number): - data.append(number) - - await clunk_lock.run() - - clunk_lock.enqueue(append(1)) - clunk_lock.enqueue(append(2)) - clunk_lock.enqueue(append(3)) - - await clunk_lock.wait() - assert data == [1, 3] - clunk_lock.kill() - - -async def test_post_queue_with_sleep(clunk_lock: ClunkLock): - data = [] - - async def append(number): - data.append(number) - await asyncio.sleep(0.01) - - await clunk_lock.run() - - clunk_lock.enqueue(append(1)) - clunk_lock.enqueue(append(2)) - clunk_lock.enqueue(append(3)) - - await clunk_lock.wait() - assert data == [1, 3] - clunk_lock.kill() - - -async def test_clunk_acquisition_eviction(clunk: Clunk): - # These should be different due to non-lazy cache eviction - r_1 = clunk.acquire("test") - r_2 = clunk.acquire("test") - assert r_1 is not r_2 - - -async def test_clunk_acquisition_different_keys(clunk: Clunk): - r_1 = clunk.acquire("test") - r_2 = clunk.acquire("test") - assert r_1 is not r_2 - - -async def test_clunk_acquisition_same(clunk: Clunk): - data = [] - - async def task(number): - data.append(number) - await asyncio.sleep(0.05) - - r_1: ClunkLock = clunk.acquire("test") - r_1.enqueue(task(1)) - await r_1.run() - - r_2: ClunkLock = clunk.acquire("test") - assert r_2.is_currently_running is True - assert r_2 is r_1 - - r_2.enqueue(task(1)) - await r_1.wait() - assert data == [1, 1] - - r_2.kill() - await asyncio.sleep(0.01) # Let the loop propagate - assert r_1.is_currently_running is False From eb03f25ea6fb13b934aeb3358ef01b4b30c0041e Mon Sep 17 00:00:00 2001 From: skelmis Date: Sun, 4 Feb 2024 22:02:17 +1300 Subject: [PATCH 20/22] chore: add note to instance_info command that it is not functional --- suggestions/cogs/help_guild_cog.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/suggestions/cogs/help_guild_cog.py b/suggestions/cogs/help_guild_cog.py index e2b218c..a81f595 100644 --- a/suggestions/cogs/help_guild_cog.py +++ b/suggestions/cogs/help_guild_cog.py @@ -71,7 +71,12 @@ async def instance_info( description="The ID of the guild you want info on." ), ): - """Retrieve information about what instance a given guild sees.""" + """Retrieve information about what instance a given guild sees. This is currently wrong.""" + await interaction.send( + ephemeral=True, content="This command currently does not work correctly." + ) + return + guild_id = int(guild_id) shard_id = self.bot.get_shard_id(guild_id) cluster_id = ( From be04a8ed47b537bbb45fdc4d4a49dc9635d50d70 Mon Sep 17 00:00:00 2001 From: skelmis Date: Mon, 5 Feb 2024 08:47:39 +1300 Subject: [PATCH 21/22] fix: Exception imports for caching --- suggestions/cogs/blacklist_cog.py | 2 +- suggestions/cogs/help_guild_cog.py | 3 --- suggestions/cogs/suggestion_cog.py | 2 +- suggestions/cogs/view_voters_cog.py | 2 +- suggestions/objects/guild_config.py | 2 +- suggestions/objects/stats/member_stats.py | 2 +- suggestions/objects/user_config.py | 2 +- 7 files changed, 6 insertions(+), 9 deletions(-) diff --git a/suggestions/cogs/blacklist_cog.py b/suggestions/cogs/blacklist_cog.py index 64f3f19..8c7c408 100644 --- a/suggestions/cogs/blacklist_cog.py +++ b/suggestions/cogs/blacklist_cog.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING import disnake -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry from disnake.ext import commands from suggestions.objects import GuildConfig, Suggestion diff --git a/suggestions/cogs/help_guild_cog.py b/suggestions/cogs/help_guild_cog.py index a81f595..a65bba6 100644 --- a/suggestions/cogs/help_guild_cog.py +++ b/suggestions/cogs/help_guild_cog.py @@ -3,11 +3,8 @@ import datetime import io import logging -import os -import typing from typing import TYPE_CHECKING, Optional -import aiohttp import disnake from alaric import AQ from alaric.comparison import EQ diff --git a/suggestions/cogs/suggestion_cog.py b/suggestions/cogs/suggestion_cog.py index 5bef9f0..190185d 100644 --- a/suggestions/cogs/suggestion_cog.py +++ b/suggestions/cogs/suggestion_cog.py @@ -5,7 +5,7 @@ import cooldowns import disnake -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry from bot_base.wraps import WrappedChannel from disnake import Guild, Localized from disnake.ext import commands, components diff --git a/suggestions/cogs/view_voters_cog.py b/suggestions/cogs/view_voters_cog.py index 36b970b..7ec3f38 100644 --- a/suggestions/cogs/view_voters_cog.py +++ b/suggestions/cogs/view_voters_cog.py @@ -5,7 +5,7 @@ import cooldowns import disnake -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry from disnake.ext import commands from bot_base.paginators.disnake_paginator import DisnakePaginator diff --git a/suggestions/objects/guild_config.py b/suggestions/objects/guild_config.py index 389386d..1f1b803 100644 --- a/suggestions/objects/guild_config.py +++ b/suggestions/objects/guild_config.py @@ -5,7 +5,7 @@ from alaric import AQ from alaric.comparison import EQ -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry if TYPE_CHECKING: from suggestions import State diff --git a/suggestions/objects/stats/member_stats.py b/suggestions/objects/stats/member_stats.py index f20cd6b..fed9334 100644 --- a/suggestions/objects/stats/member_stats.py +++ b/suggestions/objects/stats/member_stats.py @@ -7,7 +7,7 @@ from alaric import AQ from alaric.comparison import EQ from alaric.logical import AND -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry from .member_command_stats import MemberCommandStats diff --git a/suggestions/objects/user_config.py b/suggestions/objects/user_config.py index 78c885d..580f556 100644 --- a/suggestions/objects/user_config.py +++ b/suggestions/objects/user_config.py @@ -5,7 +5,7 @@ from alaric import AQ from alaric.comparison import EQ -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry if TYPE_CHECKING: from suggestions import State From 333897dda74ff70252ba0954b72a0f3e156c5b97 Mon Sep 17 00:00:00 2001 From: Ethan <47520067+Skelmis@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:03:51 +1300 Subject: [PATCH 22/22] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5fbb6dd..70df6d3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,5 +1,4 @@ ## Summary - ## Checklist @@ -14,4 +13,4 @@ - [ ] New errors have been updated on ``stats.suggestions.gg`` - [ ] Guild config method names aren't duplicated - [ ] New localizations have been added -- [ ] Documentation on ``docs.suggestions.gg`` has been updated \ No newline at end of file +- [ ] Documentation on ``docs.suggestions.gg`` has been updated