chore: refactor garven (#72)

* feat: begin migration

* feat: move cluster status

* chore: fix imports

requirements.txt

@@ -2,26 +2,20 @@ aiodns==3.0.0
 aiohttp==3.8.1
 aiosignal==1.2.0
 alaric==1.2.0
-anyio==4.2.0
 async-timeout==4.0.2
 attrs==21.4.0
 Bot-Base==1.7.1
 Brotli==1.0.9
 causar==0.2.0
 cchardet==2.1.7
-certifi==2023.11.17
 cffi==1.15.0
 charset-normalizer==2.0.12
 disnake @ git+https://github.com/suggestionsbot/disnake.git@22a572afd139144c0e2de49c7692a73ab74d8a3d
 disnake-ext-components @ git+https://github.com/suggestionsbot/disnake-ext-components.git@91689ed74ffee73f631453a39e548af9b824826d
 dnspython==2.2.1
-exceptiongroup==1.2.0
 frozenlist==1.3.0
 function-cooldowns==1.3.1
 graphviz==0.20.1
-h11==0.14.0
-httpcore==1.0.2
-httpx==0.26.0
 humanize==4.2.0
 idna==3.3
 iniconfig==1.1.1
@@ -44,7 +38,6 @@ pytest==7.1.3
 pytest-asyncio==0.19.0
 python-dotenv==0.20.0
 sentinels==1.0.0
-sniffio==1.3.0
 tomli==2.0.1
 typing_extensions==4.3.0
 websockets==10.4

suggestions/bot.py

@@ -34,6 +34,7 @@
     UnhandledError,
     QueueImbalance,
     BlocklistedUser,
+    PartialResponse,
 )
 from suggestions.http_error_parser import try_parse_http_error
 from suggestions.objects import Error, GuildConfig, UserConfig
@@ -652,38 +653,25 @@ async def process_update_bot_listings():
 
             headers = {"Authorization": os.environ["SUGGESTIONS_API_KEY"]}
             while not state.is_closing:
-                url = (
-                    "https://garven.suggestions.gg/aggregate/guilds/count"
-                    if self.is_prod
-                    else "https://garven.dev.suggestions.gg/aggregate/guilds/count"
-                )
-                async with aiohttp.ClientSession(
-                    headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]}
-                ) as session:
-                    async with session.get(url) as resp:
-                        data: dict = await resp.json()
-                        if resp.status != 200:
-                            log.error("Stopping bot list updates")
-                            log.error("%s", data)
-                            break
-
-                if data["partial_response"]:
-                    log.warning(
-                        "Skipping bot list updates as IPC returned a partial responses"
-                    )
+                try:
+                    total_guilds = await self.garven.get_total_guilds()
+                except PartialResponse:
                     await self.sleep_with_condition(
                         time_between_updates.total_seconds(),
                         lambda: self.state.is_closing,
                     )
                     continue
 
                 body = {
-                    "guild_count": int(data["statistic"]),
+                    "guild_count": int(total_guilds),
                     "shard_count": int(self.shard_count),
                 }
                 async with aiohttp.ClientSession(headers=headers) as session:
                     async with session.post(
-                        os.environ["SUGGESTIONS_STATS_API_URL"], json=body
+                        os.environ[
+                            "SUGGESTIONS_STATS_API_URL"
+                        ],  # This is the bot list API # lists.suggestions.gg
+                        json=body,
                     ) as r:
                         if r.status != 200:
                             log.warning("%s", r.text)
@@ -834,13 +822,14 @@ async def inner():
                         log.error("Borked it")
                         return
 
+                    tb = "".join(traceback.format_exception(e))
                     log.error(
                         "Status update failed: %s",
-                        "".join(traceback.format_exception(e)),
+                        tb,
                     )
                     await self.garven.notify_devs(
                         title="Status page ping error",
-                        description=str(e),
+                        description=tb,
                         sender=f"Cluster {self.cluster_id}, shard {self.shard_id}",
                     )
 

suggestions/cogs/help_guild_cog.py

@@ -143,11 +143,6 @@ async def show_bot_status(
 
         red_circle = "🔴"
         green_circle = "🟢"
-        url = (
-            "https://garven.suggestions.gg/cluster/status"
-            if self.bot.is_prod
-            else "https://garven.dev.suggestions.gg/cluster/status"
-        )
 
         embed = disnake.Embed(
             timestamp=datetime.datetime.utcnow(),
@@ -156,13 +151,7 @@ async def show_bot_status(
         down_shards: list[str] = [str(i) for i in range(53)]
         down_clusters: list[str] = [str(i) for i in range(1, 7)]
         avg_bot_latency: list[float] = []
-        async with aiohttp.ClientSession(
-            headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]}
-        ) as session:
-            async with session.get(url) as resp:
-                data: dict[str, dict | bool] = await resp.json()
-                if resp.status != 200:
-                    log.error("Something went wrong: %s", data)
+        data = await self.bot.garven.cluster_status()
 
         if data.pop("partial_response") is not None:
             embed.set_footer(text="Partial response")

suggestions/exceptions.py

@@ -49,3 +49,7 @@ class QueueImbalance(disnake.DiscordException):
 
 class BlocklistedUser(CheckFailure):
     """This user is blocked from taking this action in this guild."""
+
+
+class PartialResponse(Exception):
+    """A garven route returned a partial response when we require a full response"""

suggestions/garven.py

@@ -6,6 +6,8 @@
 
 import aiohttp
 
+from suggestions.exceptions import PartialResponse
+
 if TYPE_CHECKING:
     from suggestions import SuggestionsBot
 
@@ -19,12 +21,30 @@ def __init__(self, bot: SuggestionsBot):
             if bot.is_prod
             else "https://garven.dev.suggestions.gg"
         )
+        self._ws_url = (
+            "wss://garven.suggestions.gg/ws"
+            if bot.is_prod
+            else "wss://garven.dev.suggestions.gg/ws"
+        )
         self._session: aiohttp.ClientSession = aiohttp.ClientSession(
             base_url=self._url,
             headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]},
         )
         self.bot: SuggestionsBot = bot
 
+    @property
+    def http_url(self) -> str:
+        return self._url
+
+    @property
+    def ws_url(self) -> str:
+        return self._ws_url
+
+    @staticmethod
+    async def _handle_status(resp: aiohttp.ClientResponse):
+        if resp.status > 299:
+            raise ValueError(f"Garven route failed {resp.url}")
+
     async def notify_devs(self, *, title: str, description: str, sender: str):
         async with self._session.post(
             "/cluster/notify_devs",
@@ -39,3 +59,29 @@ async def notify_devs(self, *, title: str, description: str, sender: str):
                     "Error when attempting to notify devs\n\t- %s",
                     await resp.text(),
                 )
+
+    async def get_shard_info(self, guild_id: int) -> dict[str, str]:
+        async with self._session.get(
+            f"/aggregate/guilds/{guild_id}/shard_info"
+        ) as resp:
+            await self._handle_status(resp)
+            data = await resp.json()
+
+        return data
+
+    async def get_total_guilds(self) -> int:
+        async with self._session.get("/aggregate/guilds/count") as resp:
+            await self._handle_status(resp)
+            data = await resp.json()
+            if data["partial_response"]:
+                log.warning("get_total_guilds returned a partial response")
+                raise PartialResponse
+
+        return data["statistic"]
+
+    async def cluster_status(self) -> dict:
+        async with self._session.get("/cluster/status") as resp:
+            await self._handle_status(resp)
+            data = await resp.json()
+
+        return data

suggestions/main.py

@@ -9,10 +9,8 @@
 import textwrap
 from traceback import format_exception
 
-import aiohttp
 import cooldowns
 import disnake
-import httpx
 from disnake import Locale
 from disnake.ext import commands
 from bot_base.paginators.disnake_paginator import DisnakePaginator
@@ -28,9 +26,6 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot:
     is_prod: bool = True if os.environ.get("PROD", None) else False
 
     if is_prod:
-        # TODO Fix this
-        # request = httpx.get("http://localhost:7878/shard-count")
-        # total_shards = int(request.text)
         total_shards = int(os.environ["TOTAL_SHARDS"])
         cluster_id = int(os.environ["CLUSTER"])
         offset = cluster_id - 1

suggestions/zonis_routes.py

@@ -19,13 +19,8 @@
 class ZonisRoutes:
     def __init__(self, bot: SuggestionsBot):
         self.bot: SuggestionsBot = bot
-        url = (
-            "wss://garven.suggestions.gg/ws"
-            if self.bot.is_prod
-            else "wss://garven.dev.suggestions.gg/ws"
-        )
         self.client: client.Client = client.Client(
-            url=url,
+            url=bot.garven.ws_url,
             identifier=str(bot.cluster_id),
             secret_key=os.environ["ZONIS_SECRET_KEY"],
             override_key=os.environ.get("ZONIS_OVERRIDE_KEY"),