From 0195bd125f060b50dd8cf7bbcca82edb420f69f9 Mon Sep 17 00:00:00 2001 From: Ethan <47520067+Skelmis@users.noreply.github.com> Date: Sat, 6 Jan 2024 19:29:02 +1300 Subject: [PATCH] chore: refactor garven (#72) * feat: begin migration * feat: move cluster status * chore: fix imports --- requirements.txt | 7 ----- suggestions/bot.py | 35 ++++++++--------------- suggestions/cogs/help_guild_cog.py | 13 +-------- suggestions/exceptions.py | 4 +++ suggestions/garven.py | 46 ++++++++++++++++++++++++++++++ suggestions/main.py | 5 ---- suggestions/zonis_routes.py | 7 +---- 7 files changed, 64 insertions(+), 53 deletions(-) diff --git a/requirements.txt b/requirements.txt index a3ff0d5..558ece6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,26 +2,20 @@ aiodns==3.0.0 aiohttp==3.8.1 aiosignal==1.2.0 alaric==1.2.0 -anyio==4.2.0 async-timeout==4.0.2 attrs==21.4.0 Bot-Base==1.7.1 Brotli==1.0.9 causar==0.2.0 cchardet==2.1.7 -certifi==2023.11.17 cffi==1.15.0 charset-normalizer==2.0.12 disnake @ git+https://github.com/suggestionsbot/disnake.git@22a572afd139144c0e2de49c7692a73ab74d8a3d disnake-ext-components @ git+https://github.com/suggestionsbot/disnake-ext-components.git@91689ed74ffee73f631453a39e548af9b824826d dnspython==2.2.1 -exceptiongroup==1.2.0 frozenlist==1.3.0 function-cooldowns==1.3.1 graphviz==0.20.1 -h11==0.14.0 -httpcore==1.0.2 -httpx==0.26.0 humanize==4.2.0 idna==3.3 iniconfig==1.1.1 @@ -44,7 +38,6 @@ pytest==7.1.3 pytest-asyncio==0.19.0 python-dotenv==0.20.0 sentinels==1.0.0 -sniffio==1.3.0 tomli==2.0.1 typing_extensions==4.3.0 websockets==10.4 diff --git a/suggestions/bot.py b/suggestions/bot.py index d7b14ab..4343e53 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -34,6 +34,7 @@ UnhandledError, QueueImbalance, BlocklistedUser, + PartialResponse, ) from suggestions.http_error_parser import try_parse_http_error from suggestions.objects import Error, GuildConfig, UserConfig @@ -652,25 +653,9 @@ async def process_update_bot_listings(): headers = {"Authorization": os.environ["SUGGESTIONS_API_KEY"]} while not state.is_closing: - url = ( - "https://garven.suggestions.gg/aggregate/guilds/count" - if self.is_prod - else "https://garven.dev.suggestions.gg/aggregate/guilds/count" - ) - async with aiohttp.ClientSession( - headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]} - ) as session: - async with session.get(url) as resp: - data: dict = await resp.json() - if resp.status != 200: - log.error("Stopping bot list updates") - log.error("%s", data) - break - - if data["partial_response"]: - log.warning( - "Skipping bot list updates as IPC returned a partial responses" - ) + try: + total_guilds = await self.garven.get_total_guilds() + except PartialResponse: await self.sleep_with_condition( time_between_updates.total_seconds(), lambda: self.state.is_closing, @@ -678,12 +663,15 @@ async def process_update_bot_listings(): continue body = { - "guild_count": int(data["statistic"]), + "guild_count": int(total_guilds), "shard_count": int(self.shard_count), } async with aiohttp.ClientSession(headers=headers) as session: async with session.post( - os.environ["SUGGESTIONS_STATS_API_URL"], json=body + os.environ[ + "SUGGESTIONS_STATS_API_URL" + ], # This is the bot list API # lists.suggestions.gg + json=body, ) as r: if r.status != 200: log.warning("%s", r.text) @@ -834,13 +822,14 @@ async def inner(): log.error("Borked it") return + tb = "".join(traceback.format_exception(e)) log.error( "Status update failed: %s", - "".join(traceback.format_exception(e)), + tb, ) await self.garven.notify_devs( title="Status page ping error", - description=str(e), + description=tb, sender=f"Cluster {self.cluster_id}, shard {self.shard_id}", ) diff --git a/suggestions/cogs/help_guild_cog.py b/suggestions/cogs/help_guild_cog.py index 4b1d602..e2b218c 100644 --- a/suggestions/cogs/help_guild_cog.py +++ b/suggestions/cogs/help_guild_cog.py @@ -143,11 +143,6 @@ async def show_bot_status( red_circle = "🔴" green_circle = "🟢" - url = ( - "https://garven.suggestions.gg/cluster/status" - if self.bot.is_prod - else "https://garven.dev.suggestions.gg/cluster/status" - ) embed = disnake.Embed( timestamp=datetime.datetime.utcnow(), @@ -156,13 +151,7 @@ async def show_bot_status( down_shards: list[str] = [str(i) for i in range(53)] down_clusters: list[str] = [str(i) for i in range(1, 7)] avg_bot_latency: list[float] = [] - async with aiohttp.ClientSession( - headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]} - ) as session: - async with session.get(url) as resp: - data: dict[str, dict | bool] = await resp.json() - if resp.status != 200: - log.error("Something went wrong: %s", data) + data = await self.bot.garven.cluster_status() if data.pop("partial_response") is not None: embed.set_footer(text="Partial response") diff --git a/suggestions/exceptions.py b/suggestions/exceptions.py index 1e78006..6669a63 100644 --- a/suggestions/exceptions.py +++ b/suggestions/exceptions.py @@ -49,3 +49,7 @@ class QueueImbalance(disnake.DiscordException): class BlocklistedUser(CheckFailure): """This user is blocked from taking this action in this guild.""" + + +class PartialResponse(Exception): + """A garven route returned a partial response when we require a full response""" diff --git a/suggestions/garven.py b/suggestions/garven.py index a2891c4..2dabf32 100644 --- a/suggestions/garven.py +++ b/suggestions/garven.py @@ -6,6 +6,8 @@ import aiohttp +from suggestions.exceptions import PartialResponse + if TYPE_CHECKING: from suggestions import SuggestionsBot @@ -19,12 +21,30 @@ def __init__(self, bot: SuggestionsBot): if bot.is_prod else "https://garven.dev.suggestions.gg" ) + self._ws_url = ( + "wss://garven.suggestions.gg/ws" + if bot.is_prod + else "wss://garven.dev.suggestions.gg/ws" + ) self._session: aiohttp.ClientSession = aiohttp.ClientSession( base_url=self._url, headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]}, ) self.bot: SuggestionsBot = bot + @property + def http_url(self) -> str: + return self._url + + @property + def ws_url(self) -> str: + return self._ws_url + + @staticmethod + async def _handle_status(resp: aiohttp.ClientResponse): + if resp.status > 299: + raise ValueError(f"Garven route failed {resp.url}") + async def notify_devs(self, *, title: str, description: str, sender: str): async with self._session.post( "/cluster/notify_devs", @@ -39,3 +59,29 @@ async def notify_devs(self, *, title: str, description: str, sender: str): "Error when attempting to notify devs\n\t- %s", await resp.text(), ) + + async def get_shard_info(self, guild_id: int) -> dict[str, str]: + async with self._session.get( + f"/aggregate/guilds/{guild_id}/shard_info" + ) as resp: + await self._handle_status(resp) + data = await resp.json() + + return data + + async def get_total_guilds(self) -> int: + async with self._session.get("/aggregate/guilds/count") as resp: + await self._handle_status(resp) + data = await resp.json() + if data["partial_response"]: + log.warning("get_total_guilds returned a partial response") + raise PartialResponse + + return data["statistic"] + + async def cluster_status(self) -> dict: + async with self._session.get("/cluster/status") as resp: + await self._handle_status(resp) + data = await resp.json() + + return data diff --git a/suggestions/main.py b/suggestions/main.py index 12217bc..0f8c30c 100644 --- a/suggestions/main.py +++ b/suggestions/main.py @@ -9,10 +9,8 @@ import textwrap from traceback import format_exception -import aiohttp import cooldowns import disnake -import httpx from disnake import Locale from disnake.ext import commands from bot_base.paginators.disnake_paginator import DisnakePaginator @@ -28,9 +26,6 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot: is_prod: bool = True if os.environ.get("PROD", None) else False if is_prod: - # TODO Fix this - # request = httpx.get("http://localhost:7878/shard-count") - # total_shards = int(request.text) total_shards = int(os.environ["TOTAL_SHARDS"]) cluster_id = int(os.environ["CLUSTER"]) offset = cluster_id - 1 diff --git a/suggestions/zonis_routes.py b/suggestions/zonis_routes.py index 5e9b284..0e7c359 100644 --- a/suggestions/zonis_routes.py +++ b/suggestions/zonis_routes.py @@ -19,13 +19,8 @@ class ZonisRoutes: def __init__(self, bot: SuggestionsBot): self.bot: SuggestionsBot = bot - url = ( - "wss://garven.suggestions.gg/ws" - if self.bot.is_prod - else "wss://garven.dev.suggestions.gg/ws" - ) self.client: client.Client = client.Client( - url=url, + url=bot.garven.ws_url, identifier=str(bot.cluster_id), secret_key=os.environ["ZONIS_SECRET_KEY"], override_key=os.environ.get("ZONIS_OVERRIDE_KEY"),