diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5fbb6dd..70df6d3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,5 +1,4 @@ ## Summary - ## Checklist @@ -14,4 +13,4 @@ - [ ] New errors have been updated on ``stats.suggestions.gg`` - [ ] Guild config method names aren't duplicated - [ ] New localizations have been added -- [ ] Documentation on ``docs.suggestions.gg`` has been updated \ No newline at end of file +- [ ] Documentation on ``docs.suggestions.gg`` has been updated diff --git a/requirements.txt b/requirements.txt index 28afb05..b64820a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,20 +2,26 @@ aiodns==3.0.0 aiohttp==3.8.1 aiosignal==1.2.0 alaric==1.2.0 +anyio==4.2.0 async-timeout==4.0.2 attrs==21.4.0 Bot-Base==1.7.1 Brotli==1.0.9 causar==0.2.0 cchardet==2.1.7 +certifi==2023.11.17 cffi==1.15.0 charset-normalizer==2.0.12 -disnake @ git+https://github.com/suggestionsbot/disnake.git@e489a5cd5561269a41a76b5037a38993886d7bfd +disnake @ git+https://github.com/suggestionsbot/disnake.git@22a572afd139144c0e2de49c7692a73ab74d8a3d disnake-ext-components @ git+https://github.com/suggestionsbot/disnake-ext-components.git@91689ed74ffee73f631453a39e548af9b824826d dnspython==2.2.1 +exceptiongroup==1.2.0 frozenlist==1.3.0 function-cooldowns==1.3.1 graphviz==0.20.1 +h11==0.14.0 +httpcore==1.0.2 +httpx==0.26.0 humanize==4.2.0 idna==3.3 iniconfig==1.1.1 @@ -38,6 +44,8 @@ pytest==7.1.3 pytest-asyncio==0.19.0 python-dotenv==0.20.0 sentinels==1.0.0 +skelmis-commons==1.1.0 +sniffio==1.3.0 tomli==2.0.1 typing_extensions==4.3.0 websockets==10.4 diff --git a/suggestions/bot.py b/suggestions/bot.py index 6ec89c5..594ffb4 100644 --- a/suggestions/bot.py +++ b/suggestions/bot.py @@ -2,6 +2,8 @@ import asyncio import datetime +import gc +import io import logging import math import os @@ -16,12 +18,11 @@ from alaric import Cursor from bot_base.wraps import WrappedChannel from cooldowns import CallableOnCooldown -from disnake import Locale, LocalizationKeyError +from disnake import Locale, LocalizationKeyError, GatewayParams from disnake.ext import commands from bot_base import BotBase, BotContext, PrefixNotFound from suggestions import State, Colors, Emojis, ErrorCode, Garven -from suggestions.clunk import Clunk from suggestions.exceptions import ( BetaOnly, MissingSuggestionsChannel, @@ -34,6 +35,7 @@ UnhandledError, QueueImbalance, BlocklistedUser, + PartialResponse, ) from suggestions.http_error_parser import try_parse_http_error from suggestions.objects import Error, GuildConfig, UserConfig @@ -46,7 +48,7 @@ class SuggestionsBot(commands.AutoShardedInteractionBot, BotBase): def __init__(self, *args, **kwargs): - self.version: str = "Public Release 3.19" + self.version: str = "Public Release 3.21" self.main_guild_id: int = 601219766258106399 self.legacy_beta_role_id: int = 995588041991274547 self.automated_beta_role_id: int = 998173237282361425 @@ -72,7 +74,6 @@ def __init__(self, *args, **kwargs): self.state: State = State(self.db, self) self.stats: Stats = Stats(self) self.garven: Garven = Garven(self) - self.clunk: Clunk = Clunk(self.state) self.suggestion_emojis: Emojis = Emojis(self) self.old_prefixed_commands: set[str] = { "changelog", @@ -89,6 +90,7 @@ def __init__(self, *args, **kwargs): "vote", } self.converted_prefix_commands: set[str] = {"suggest", "approve", "reject"} + self.gc_lock: asyncio.Lock = asyncio.Lock() # Sharding info self.cluster_id: int = kwargs.pop("cluster", 0) @@ -103,6 +105,7 @@ def __init__(self, *args, **kwargs): name="suggestions", type=disnake.ActivityType.watching, ), + # gateway_params=GatewayParams(zlib=False), ) self._has_dispatched_initial_ready: bool = False @@ -110,6 +113,19 @@ def __init__(self, *args, **kwargs): self.zonis: ZonisRoutes = ZonisRoutes(self) + async def launch_shard( + self, _gateway: str, shard_id: int, *, initial: bool = False + ) -> None: + # Use the proxy if set, else fall back to whatever is default + proxy: Optional[str] = os.environ.get("GW_PROXY", _gateway) + return await super().launch_shard(proxy, shard_id, initial=initial) + + async def before_identify_hook( + self, _shard_id: int | None, *, initial: bool = False # noqa: ARG002 + ) -> None: + # gateway-proxy + return + async def get_or_fetch_channel(self, channel_id: int) -> WrappedChannel: try: return await super().get_or_fetch_channel(channel_id) @@ -126,14 +142,19 @@ async def dispatch_initial_ready(self): log.info("Startup took: %s", self.get_uptime()) await self.suggestion_emojis.populate_emojis() + async def on_resumed(self): + if self.gc_lock.locked(): + return + + async with self.gc_lock: + await asyncio.sleep(2.0) + collected = gc.collect() + log.info(f"Garbage collector: collected {collected} objects.") + @property def total_cluster_count(self) -> int: return math.ceil(self.total_shards / 10) - @property - def is_primary_cluster(self) -> bool: - return bool(os.environ.get("IS_PRIMARY_CLUSTER", False)) - def error_embed( self, title: str, @@ -560,6 +581,7 @@ async def load(self): await self.stats.load() await self.update_bot_listings() await self.push_status() + await self.update_dev_channel() await self.watch_for_shutdown_request() await self.load_cogs() await self.zonis.start() @@ -571,7 +593,6 @@ async def graceful_shutdown(self) -> None: """ log.debug("Attempting to shutdown") self.state.notify_shutdown() - await self.clunk.kill_all() await self.zonis.client.close() await asyncio.gather(*self.state.background_tasks) log.info("Shutting down") @@ -624,6 +645,65 @@ async def process_watch_for_shutdown(): process_watch_for_shutdown.__task = task_1 state.add_background_task(task_1) + async def update_dev_channel(self): + if not self.is_prod: + log.info("Not watching for debug info as not on prod") + return + + if not self.is_primary_cluster: + log.info("Not watching for debug info as not primary cluster") + return + + state: State = self.state + + async def process_watch_for_shutdown(): + await self.wait_until_ready() + log.debug("Started tracking bot latency") + + while not state.is_closing: + # Update once an hour + await self.sleep_with_condition( + datetime.timedelta(minutes=5).total_seconds(), + lambda: self.state.is_closing, + ) + + await self.garven.notify_devs( + title=f"WS latency as follows", + description=f"Timestamped for {datetime.datetime.utcnow().isoformat()}", + sender=f"N/A", + ) + + data = await self.garven.get_bot_ws_latency() + shard_data = data["shards"] + for i in range(0, 75, 5): + description = io.StringIO() + for o in range(0, 6): + shard = str(i + o) + try: + description.write( + f"**Shard {shard}**\nWS latency: `{shard_data[shard]['ws']}`\n" + f"Keep Alive latency: `{shard_data[shard]['keepalive']}`\n\n" + ) + except KeyError: + # My lazy way of not doing env checks n math right + continue + + if description.getvalue(): + await self.garven.notify_devs( + title=f"WS latency", + description=description.getvalue(), + sender=f"Partial response: {data['partial_response']}", + ) + + await self.sleep_with_condition( + datetime.timedelta(hours=1).total_seconds(), + lambda: self.state.is_closing, + ) + + task_1 = asyncio.create_task(process_watch_for_shutdown()) + process_watch_for_shutdown.__task = task_1 + state.add_background_task(task_1) + async def update_bot_listings(self) -> None: """Updates the bot lists with current stats.""" if not self.is_prod: @@ -642,25 +722,9 @@ async def process_update_bot_listings(): headers = {"Authorization": os.environ["SUGGESTIONS_API_KEY"]} while not state.is_closing: - url = ( - "https://garven.suggestions.gg/aggregate/guilds/count" - if self.is_prod - else "https://garven.dev.suggestions.gg/aggregate/guilds/count" - ) - async with aiohttp.ClientSession( - headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]} - ) as session: - async with session.get(url) as resp: - data: dict = await resp.json() - if resp.status != 200: - log.error("Stopping bot list updates") - log.error("%s", data) - break - - if data["partial_response"]: - log.warning( - "Skipping bot list updates as IPC returned a partial responses" - ) + try: + total_guilds = await self.garven.get_total_guilds() + except PartialResponse: await self.sleep_with_condition( time_between_updates.total_seconds(), lambda: self.state.is_closing, @@ -668,12 +732,15 @@ async def process_update_bot_listings(): continue body = { - "guild_count": int(data["statistic"]), + "guild_count": int(total_guilds), "shard_count": int(self.shard_count), } async with aiohttp.ClientSession(headers=headers) as session: async with session.post( - os.environ["SUGGESTIONS_STATS_API_URL"], json=body + os.environ[ + "SUGGESTIONS_STATS_API_URL" + ], # This is the bot list API # lists.suggestions.gg + json=body, ) as r: if r.status != 200: log.warning("%s", r.text) @@ -688,6 +755,24 @@ async def process_update_bot_listings(): state.add_background_task(task_1) log.info("Setup bot list updates") + @property + def is_primary_cluster(self) -> bool: + if not self.is_prod: + # Non-prod is always single cluster + return True + + shard_id = self.get_shard_id(self.main_guild_id) + return shard_id in self.shard_ids + + async def _sync_application_commands(self) -> None: + # In order to reduce getting rate-limited because every cluster + # decided it wants to sync application commands when it aint required + if not self.is_primary_cluster: + log.warning("Not syncing application commands as not primary cluster") + return + + await super()._sync_application_commands() + def get_shard_id(self, guild_id: Optional[int]) -> int: # DM's go to shard 0 shard_id = 0 @@ -806,13 +891,14 @@ async def inner(): log.error("Borked it") return + tb = "".join(traceback.format_exception(e)) log.error( "Status update failed: %s", - "".join(traceback.format_exception(e)), + tb, ) await self.garven.notify_devs( title="Status page ping error", - description=str(e), + description=tb, sender=f"Cluster {self.cluster_id}, shard {self.shard_id}", ) diff --git a/suggestions/clunk/__init__.py b/suggestions/clunk/__init__.py deleted file mode 100644 index 6bbec54..0000000 --- a/suggestions/clunk/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .lock import ClunkLock -from .cache import ClunkCache -from .clunk import Clunk diff --git a/suggestions/clunk/cache.py b/suggestions/clunk/cache.py deleted file mode 100644 index 7d2258a..0000000 --- a/suggestions/clunk/cache.py +++ /dev/null @@ -1,25 +0,0 @@ -from bot_base.caches import TimedCache - - -class ClunkCache(TimedCache): - def __contains__(self, item): - try: - entry = self.cache[item] - if not entry.value.has_requests: - entry.value.kill() - self.cache.pop(item) - return False - except KeyError: - return False - else: - return True - - def force_clean(self) -> None: - items = {} - for k, v in self.cache.items(): - if v.value.has_requests: - items[k] = v - else: - v.value.kill() - - self.cache = items diff --git a/suggestions/clunk/clunk.py b/suggestions/clunk/clunk.py deleted file mode 100644 index cdfb6da..0000000 --- a/suggestions/clunk/clunk.py +++ /dev/null @@ -1,32 +0,0 @@ -import asyncio -import logging -from datetime import timedelta - -from bot_base import NonExistentEntry - -from suggestions import State -from suggestions.clunk import ClunkCache, ClunkLock - -log = logging.getLogger(__name__) - - -class Clunk: - def __init__(self, state: State): - self._state: State = state - self._cache: ClunkCache[str, ClunkLock] = ClunkCache( - lazy_eviction=False, global_ttl=timedelta(hours=1) - ) - - def acquire(self, suggestion_id: str) -> ClunkLock: - key = suggestion_id - try: - return self._cache.get_entry(key) - except NonExistentEntry: - lock = ClunkLock(self._state) - self._cache.add_entry(key, lock) - return lock - - async def kill_all(self) -> None: - """Kill all current ClunkLock instances.""" - for lock in self._cache.cache.values(): - lock.value.kill() diff --git a/suggestions/clunk/lock.py b/suggestions/clunk/lock.py deleted file mode 100644 index 5886b18..0000000 --- a/suggestions/clunk/lock.py +++ /dev/null @@ -1,124 +0,0 @@ -import asyncio -from typing import Coroutine - -from suggestions import State - - -class ClunkLock: - """Custom request processing for the 21st century. - - Example: - - We need to edit a message 9 times. - Each time the message is edited we simply increment a counter. - - For bulk operations this causes rate-limits to be hit - as we process the edits one by one until we exhaust the requests. - - Given a FIFO queue however, we can in theory skip requests - to reduce the amount of edits required to reach the same result. - - For example, the below two approaches would result in the same - output for the end user but the proposed design would only - make two requests instead of the nine requests it currently takes - which should alleviate errors we run into due to rate-limits. - - Currently: - +---+---+---+---+---+---+---+---+---+ - | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | - +---+---+---+---+---+---+---+---+---+ - - Proposed: - +---+---+ - | 1 | 9 | - +---+---+ - - Find the generic version here: https://workbin.dev/?id=1664955297537152 - """ - - def __init__(self, state: State): - self._state: State = state - self.__killed: bool = False - self.is_currently_running: bool = False - self._next_request: Coroutine | None = None - self._event: asyncio.Event = asyncio.Event() - self._current_request: Coroutine | None = None - - @property - def has_requests(self) -> None: - return self._current_request is not None - - @property - def __is_closing(self) -> bool: - return self._state.is_closing or self.__killed - - async def wait(self) -> None: - """Block until all requests are processed.""" - while self.has_requests: - await asyncio.sleep(0) - - def kill(self) -> None: - """Kill any current requests inline with a graceful shutdown.""" - self.__killed = True - if not self._event.is_set(): - self._event.set() - - async def run(self) -> None: - """Begin processing the queue in the background. - - Notes - ----- - The program lifetime must be longer then the - lifetime of the tasks enqueued otherwise tasks - may fail to be awaited before the program dies. - """ - if self.is_currently_running: - return - - asyncio.create_task(self._run()) - self.is_currently_running = True - - async def _run(self) -> None: - while not self.__is_closing: - await self._event.wait() - if self.__is_closing: - break - - await self._current_request - - if self._next_request: - self._current_request = self._next_request - self._next_request = None - else: - self._current_request = None - self._event.clear() - - self.is_currently_running = False - # Cancel any remaining coros to supress - # warnings since we want to shut this down - if self._current_request: - asyncio.create_task(self._current_request).cancel() - if self._next_request: - asyncio.create_task(self._next_request).cancel() - - def enqueue(self, request: Coroutine) -> None: - """Add a request to the queue for processing - - Parameters - ---------- - request: Coroutine - The request to queue for processing - """ - if self._current_request: - # We are already processing a request - # so queue this for future running - if self._next_request: - # Cancel the old coroutine to supress warnings - # about 'func' was never awaited - asyncio.create_task(self._next_request).cancel() - self._next_request = request - else: - # We aren't processing any requests - # so queue this for immediate running - self._current_request = request - self._event.set() diff --git a/suggestions/clunk2/edits.py b/suggestions/clunk2/edits.py index 9014983..9149c21 100644 --- a/suggestions/clunk2/edits.py +++ b/suggestions/clunk2/edits.py @@ -29,6 +29,16 @@ async def update_suggestion_message( pending_edits.add(suggestion.suggestion_id) await asyncio.sleep(time_after) + if suggestion.channel_id is None or suggestion.message_id is None: + log.debug( + "Suggestion %s had a NoneType by the time it was to be edited channel_id=%s, message_id=%s", + suggestion.suggestion_id, + suggestion.channel_id, + suggestion.message_id, + ) + pending_edits.discard(suggestion.suggestion_id) + return + try: await MessageEditing( bot, channel_id=suggestion.channel_id, message_id=suggestion.message_id diff --git a/suggestions/cogs/blacklist_cog.py b/suggestions/cogs/blacklist_cog.py index 64f3f19..8c7c408 100644 --- a/suggestions/cogs/blacklist_cog.py +++ b/suggestions/cogs/blacklist_cog.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING import disnake -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry from disnake.ext import commands from suggestions.objects import GuildConfig, Suggestion diff --git a/suggestions/cogs/help_guild_cog.py b/suggestions/cogs/help_guild_cog.py index 55f210e..a65bba6 100644 --- a/suggestions/cogs/help_guild_cog.py +++ b/suggestions/cogs/help_guild_cog.py @@ -3,11 +3,8 @@ import datetime import io import logging -import os -import typing from typing import TYPE_CHECKING, Optional -import aiohttp import disnake from alaric import AQ from alaric.comparison import EQ @@ -71,31 +68,26 @@ async def instance_info( description="The ID of the guild you want info on." ), ): - """Retrieve information about what instance a given guild sees.""" + """Retrieve information about what instance a given guild sees. This is currently wrong.""" + await interaction.send( + ephemeral=True, content="This command currently does not work correctly." + ) + return + guild_id = int(guild_id) shard_id = self.bot.get_shard_id(guild_id) cluster_id = ( 1 - if shard_id < 5 - else 2 if shard_id < 10 - else 3 - if shard_id < 15 - else 4 + else 2 if shard_id < 20 - else 5 - if shard_id < 25 - else 6 + else 3 if shard_id < 30 - else 7 - if shard_id < 35 - else 8 + else 4 if shard_id < 40 - else 9 - if shard_id < 45 - else 10 + else 5 if shard_id < 50 - else 11 + else 6 ) await interaction.send( @@ -153,26 +145,15 @@ async def show_bot_status( red_circle = "🔴" green_circle = "🟢" - url = ( - "https://garven.suggestions.gg/cluster/status" - if self.bot.is_prod - else "https://garven.dev.suggestions.gg/cluster/status" - ) embed = disnake.Embed( timestamp=datetime.datetime.utcnow(), title="Bot infrastructure status", ) down_shards: list[str] = [str(i) for i in range(53)] - down_clusters: list[str] = [str(i) for i in range(1, 12)] + down_clusters: list[str] = [str(i) for i in range(1, 7)] avg_bot_latency: list[float] = [] - async with aiohttp.ClientSession( - headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]} - ) as session: - async with session.get(url) as resp: - data: dict[str, dict | bool] = await resp.json() - if resp.status != 200: - log.error("Something went wrong: %s", data) + data = await self.bot.garven.cluster_status() if data.pop("partial_response") is not None: embed.set_footer(text="Partial response") diff --git a/suggestions/cogs/suggestion_cog.py b/suggestions/cogs/suggestion_cog.py index 5bef9f0..190185d 100644 --- a/suggestions/cogs/suggestion_cog.py +++ b/suggestions/cogs/suggestion_cog.py @@ -5,7 +5,7 @@ import cooldowns import disnake -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry from bot_base.wraps import WrappedChannel from disnake import Guild, Localized from disnake.ext import commands, components diff --git a/suggestions/cogs/suggestion_queue_cog.py b/suggestions/cogs/suggestion_queue_cog.py index 98b0594..5f29c86 100644 --- a/suggestions/cogs/suggestion_queue_cog.py +++ b/suggestions/cogs/suggestion_queue_cog.py @@ -10,8 +10,7 @@ from alaric.comparison import EQ from alaric.logical import AND from alaric.projections import Projection, SHOW -from bot_base import NonExistentEntry -from bot_base.caches import TimedCache +from commons.caching import NonExistentEntry, TimedCache from disnake import Guild from disnake.ext import commands, components @@ -35,7 +34,9 @@ def __init__(self, bot): self.state = self.bot.state self.queued_suggestions_db: Document = self.bot.db.queued_suggestions self.paginator_objects: TimedCache = TimedCache( - global_ttl=timedelta(minutes=15), lazy_eviction=False + global_ttl=timedelta(minutes=15), + lazy_eviction=False, + ttl_from_last_access=True, ) async def get_paginator_for( diff --git a/suggestions/cogs/view_voters_cog.py b/suggestions/cogs/view_voters_cog.py index a184030..7ec3f38 100644 --- a/suggestions/cogs/view_voters_cog.py +++ b/suggestions/cogs/view_voters_cog.py @@ -5,13 +5,14 @@ import cooldowns import disnake -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry from disnake.ext import commands from bot_base.paginators.disnake_paginator import DisnakePaginator from suggestions import Colors from suggestions.cooldown_bucket import InteractionBucket from suggestions.objects import Suggestion +from suggestions.objects.suggestion import SuggestionState if TYPE_CHECKING: from alaric import Document @@ -109,6 +110,13 @@ async def view_suggestion_voters( channel_id=interaction.channel_id, state=self.state, ) + if suggestion.state == SuggestionState.cleared: + return await interaction.send( + self.bot.get_locale( + "VIEW_VOTERS_CLEARED_SUGGESTION", interaction.locale + ), + ephemeral=True, + ) up_vote: disnake.Emoji = await self.bot.suggestion_emojis.default_up_vote() down_vote: disnake.Emoji = await self.bot.suggestion_emojis.default_down_vote() @@ -140,6 +148,14 @@ async def view_suggestion_up_voters( channel_id=interaction.channel_id, state=self.state, ) + if suggestion.state == SuggestionState.cleared: + return await interaction.send( + self.bot.get_locale( + "VIEW_VOTERS_CLEARED_SUGGESTION", interaction.locale + ), + ephemeral=True, + ) + data = [] for voter in suggestion.up_voted_by: data.append(f"<@{voter}>") @@ -165,6 +181,14 @@ async def view_suggestion_down_voters( channel_id=interaction.channel_id, state=self.state, ) + if suggestion.state == SuggestionState.cleared: + return await interaction.send( + self.bot.get_locale( + "VIEW_VOTERS_CLEARED_SUGGESTION", interaction.locale + ), + ephemeral=True, + ) + data = [] for voter in suggestion.down_voted_by: data.append(f"<@{voter}>") @@ -203,6 +227,14 @@ async def view_voters( guild_id=interaction.guild_id, state=self.state, ) + if suggestion.state == SuggestionState.cleared: + return await interaction.send( + self.bot.get_locale( + "VIEW_VOTERS_CLEARED_SUGGESTION", interaction.locale + ), + ephemeral=True, + ) + data = [] up_vote: disnake.Emoji = await self.bot.suggestion_emojis.default_up_vote() down_vote: disnake.Emoji = await self.bot.suggestion_emojis.default_down_vote() diff --git a/suggestions/exceptions.py b/suggestions/exceptions.py index 1e78006..6669a63 100644 --- a/suggestions/exceptions.py +++ b/suggestions/exceptions.py @@ -49,3 +49,7 @@ class QueueImbalance(disnake.DiscordException): class BlocklistedUser(CheckFailure): """This user is blocked from taking this action in this guild.""" + + +class PartialResponse(Exception): + """A garven route returned a partial response when we require a full response""" diff --git a/suggestions/garven.py b/suggestions/garven.py index a2891c4..bb04931 100644 --- a/suggestions/garven.py +++ b/suggestions/garven.py @@ -6,6 +6,8 @@ import aiohttp +from suggestions.exceptions import PartialResponse + if TYPE_CHECKING: from suggestions import SuggestionsBot @@ -19,12 +21,32 @@ def __init__(self, bot: SuggestionsBot): if bot.is_prod else "https://garven.dev.suggestions.gg" ) + # self._url = "http://127.0.0.1:8002" + self._ws_url = ( + "wss://garven.suggestions.gg/ws" + if bot.is_prod + else "wss://garven.dev.suggestions.gg/ws" + ) + # self._ws_url = "ws://127.0.0.1:8002/ws" self._session: aiohttp.ClientSession = aiohttp.ClientSession( base_url=self._url, headers={"X-API-KEY": os.environ["GARVEN_API_KEY"]}, ) self.bot: SuggestionsBot = bot + @property + def http_url(self) -> str: + return self._url + + @property + def ws_url(self) -> str: + return self._ws_url + + @staticmethod + async def _handle_status(resp: aiohttp.ClientResponse): + if resp.status > 299: + raise ValueError(f"Garven route failed {resp.url}") + async def notify_devs(self, *, title: str, description: str, sender: str): async with self._session.post( "/cluster/notify_devs", @@ -39,3 +61,36 @@ async def notify_devs(self, *, title: str, description: str, sender: str): "Error when attempting to notify devs\n\t- %s", await resp.text(), ) + + async def get_shard_info(self, guild_id: int) -> dict[str, str]: + async with self._session.get( + f"/aggregate/guilds/{guild_id}/shard_info" + ) as resp: + await self._handle_status(resp) + data = await resp.json() + + return data + + async def get_total_guilds(self) -> int: + async with self._session.get("/aggregate/guilds/count") as resp: + await self._handle_status(resp) + data = await resp.json() + if data["partial_response"]: + log.warning("get_total_guilds returned a partial response") + raise PartialResponse + + return data["statistic"] + + async def cluster_status(self) -> dict: + async with self._session.get("/cluster/status") as resp: + await self._handle_status(resp) + data = await resp.json() + + return data + + async def get_bot_ws_latency(self): + async with self._session.get("/cluster/latency/ws") as resp: + await self._handle_status(resp) + data = await resp.json() + + return data diff --git a/suggestions/locales/en_GB.json b/suggestions/locales/en_GB.json index 4d3cbc5..401c717 100644 --- a/suggestions/locales/en_GB.json +++ b/suggestions/locales/en_GB.json @@ -121,5 +121,6 @@ "SUGGESTION_ID_NAME": "suggestion_id", "SUGGESTION_ID_DESCRIPTION": "The suggestions ID you wish to reference.", "USER_ID_NAME": "user_id", - "USER_ID_DESCRIPTION": "The users discord id." + "USER_ID_DESCRIPTION": "The users discord id.", + "VIEW_VOTERS_CLEARED_SUGGESTION": "Cannot view a cleared suggestion." } diff --git a/suggestions/main.py b/suggestions/main.py index 602c79c..0f8c30c 100644 --- a/suggestions/main.py +++ b/suggestions/main.py @@ -26,10 +26,10 @@ async def create_bot(database_wrapper=None) -> SuggestionsBot: is_prod: bool = True if os.environ.get("PROD", None) else False if is_prod: - total_shards = 53 + total_shards = int(os.environ["TOTAL_SHARDS"]) cluster_id = int(os.environ["CLUSTER"]) offset = cluster_id - 1 - number_of_shards_per_cluster = 5 + number_of_shards_per_cluster = int(os.environ["SHARDS_PER_CLUSTER"]) shard_ids = [ i for i in range( diff --git a/suggestions/objects/guild_config.py b/suggestions/objects/guild_config.py index 389386d..1f1b803 100644 --- a/suggestions/objects/guild_config.py +++ b/suggestions/objects/guild_config.py @@ -5,7 +5,7 @@ from alaric import AQ from alaric.comparison import EQ -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry if TYPE_CHECKING: from suggestions import State diff --git a/suggestions/objects/stats/member_stats.py b/suggestions/objects/stats/member_stats.py index f20cd6b..fed9334 100644 --- a/suggestions/objects/stats/member_stats.py +++ b/suggestions/objects/stats/member_stats.py @@ -7,7 +7,7 @@ from alaric import AQ from alaric.comparison import EQ from alaric.logical import AND -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry from .member_command_stats import MemberCommandStats diff --git a/suggestions/objects/user_config.py b/suggestions/objects/user_config.py index 78c885d..580f556 100644 --- a/suggestions/objects/user_config.py +++ b/suggestions/objects/user_config.py @@ -5,7 +5,7 @@ from alaric import AQ from alaric.comparison import EQ -from bot_base import NonExistentEntry +from commons.caching import NonExistentEntry if TYPE_CHECKING: from suggestions import State diff --git a/suggestions/state.py b/suggestions/state.py index 8de4b46..6cf20ba 100644 --- a/suggestions/state.py +++ b/suggestions/state.py @@ -12,9 +12,9 @@ from alaric import AQ from alaric.comparison import EQ from alaric.logical import AND +from alaric.meta import Negate from alaric.projections import PROJECTION, SHOW -from bot_base import NonExistentEntry -from bot_base.caches import TimedCache +from commons.caching import NonExistentEntry, TimedCache from suggestions.objects import GuildConfig, UserConfig @@ -41,17 +41,25 @@ def __init__(self, database: SuggestionsMongoManager, bot: SuggestionsBot): ) self.guild_cache_ttl: timedelta = timedelta(minutes=15) self.guild_cache: TimedCache[int, disnake.Guild] = TimedCache( - global_ttl=self.guild_cache_ttl, lazy_eviction=False + global_ttl=self.guild_cache_ttl, + lazy_eviction=False, + ttl_from_last_access=True, ) self.view_voters_cache: TimedCache[int, list[str]] = TimedCache( - global_ttl=self.autocomplete_cache_ttl, lazy_eviction=False + global_ttl=self.autocomplete_cache_ttl, + lazy_eviction=False, + ttl_from_last_access=True, ) self.guild_configs: TimedCache = TimedCache( - global_ttl=timedelta(minutes=30), lazy_eviction=False + global_ttl=timedelta(minutes=30), + lazy_eviction=False, + ttl_from_last_access=True, ) self.user_configs: TimedCache = TimedCache( - global_ttl=timedelta(minutes=30), lazy_eviction=False + global_ttl=timedelta(minutes=30), + lazy_eviction=False, + ttl_from_last_access=True, ) self.existing_error_ids: Set[str] = set() @@ -166,7 +174,7 @@ async def populate_sid_cache(self, guild_id: int) -> list: async def populate_view_voters_cache(self, guild_id: int) -> list: self.view_voters_cache.delete_entry(guild_id) data: List[Dict] = await self.database.suggestions.find_many( - AQ(EQ("guild_id", guild_id)), + AQ(AND(EQ("guild_id", guild_id), Negate(EQ("state", "cleared")))), projections=PROJECTION(SHOW("_id")), try_convert=False, ) diff --git a/suggestions/stats.py b/suggestions/stats.py index c26e52f..28ab7be 100644 --- a/suggestions/stats.py +++ b/suggestions/stats.py @@ -4,14 +4,12 @@ import datetime import logging from enum import Enum -from typing import TYPE_CHECKING, Optional, Dict, Literal, Union, Type +from typing import TYPE_CHECKING, Optional, Type import alaric from alaric import Cursor, AQ from alaric.comparison import EQ -from alaric.types import ObjectId -from bot_base.caches import TimedCache -from motor.motor_asyncio import AsyncIOMotorCollection +from commons.caching import TimedCache from suggestions.objects.stats import MemberStats, MemberCommandStats diff --git a/suggestions/zonis_routes.py b/suggestions/zonis_routes.py index 12d4f5f..4ca947b 100644 --- a/suggestions/zonis_routes.py +++ b/suggestions/zonis_routes.py @@ -3,7 +3,7 @@ import logging import math import os -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal import disnake from zonis import client @@ -19,13 +19,8 @@ class ZonisRoutes: def __init__(self, bot: SuggestionsBot): self.bot: SuggestionsBot = bot - url = ( - "wss://garven.suggestions.gg/ws" - if self.bot.is_prod - else "wss://garven.dev.suggestions.gg/ws" - ) self.client: client.Client = client.Client( - url=url, + url=bot.garven.ws_url, identifier=str(bot.cluster_id), secret_key=os.environ["ZONIS_SECRET_KEY"], override_key=os.environ.get("ZONIS_OVERRIDE_KEY"), @@ -37,6 +32,8 @@ def __init__(self, bot: SuggestionsBot): "share_with_devs", "refresh_premium", "shared_guilds", + "cached_item_count", + "cluster_ws_status", ) async def start(self): @@ -69,6 +66,25 @@ async def cluster_status(self): return data + @client.route() + async def cluster_ws_status( + self, + ) -> dict[str, dict[Literal["ws", "keepalive"], str]]: + data: dict[str, dict[Literal["ws", "keepalive"], str]] = {} + for shard_id, shard_info in self.bot.shards.items(): + shard_data: dict[Literal["ws", "keepalive"], str] = { + "ws": str(round(shard_info.latency, 5)) + } + wsc = shard_info._parent.ws._keep_alive + if wsc is None: + shard_data["keepalive"] = "None" + else: + shard_data["keepalive"] = str(round(wsc.latency, 5)) + + data[str(shard_id)] = shard_data + + return data + @client.route() async def share_with_devs(self, title, description, sender): channel: disnake.TextChannel = await self.bot.get_or_fetch_channel( # type: ignore @@ -79,3 +95,23 @@ async def share_with_devs(self, title, description, sender): ) embed.set_footer(text=f"Sender: {sender}") await channel.send(embed=embed) + + @client.route() + async def cached_item_count(self) -> dict[str, int]: + state = self.bot.state + stats = self.bot.stats + suggestions_queue_cog = self.bot.get_cog("SuggestionsQueueCog") + data = { + "state.autocomplete_cache": len(state.autocomplete_cache), + "state.guild_cache": len(state.guild_cache), + "state.view_voters_cache": len(state.view_voters_cache), + "state.guild_configs": len(state.guild_configs), + "state.user_configs": len(state.user_configs), + "stats.cluster_guild_cache": len(stats.cluster_guild_cache), + "stats.member_stats_cache": len(stats.member_stats_cache), + "suggestions_queue_cog.paginator_objects": len( + suggestions_queue_cog.paginator_objects # noqa + ), + } + + return data diff --git a/tests/conftest.py b/tests/conftest.py index 24b15b6..d63f866 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,13 +1,11 @@ import os from unittest.mock import AsyncMock -import disnake import pytest from causar import Causar, InjectionMetadata import suggestions -from suggestions.clunk import ClunkLock, Clunk from tests.mocks import MockedSuggestionsMongoManager @@ -43,13 +41,3 @@ async def injection_metadata(causar: Causar) -> InjectionMetadata: return InjectionMetadata( guild_id=881118111967883295, channel_id=causar.faker.generate_snowflake() ) - - -@pytest.fixture -async def clunk_lock(causar: Causar) -> ClunkLock: - return ClunkLock(causar.bot.state) # type: ignore - - -@pytest.fixture -async def clunk(causar: Causar) -> Clunk: - return Clunk(causar.bot.state) # type: ignore diff --git a/tests/test_clunk.py b/tests/test_clunk.py deleted file mode 100644 index a6d6625..0000000 --- a/tests/test_clunk.py +++ /dev/null @@ -1,93 +0,0 @@ -import asyncio - -from suggestions.clunk import Clunk, ClunkLock - - -async def test_pre_queue(clunk_lock: ClunkLock): - data = [] - - async def append(number): - data.append(number) - - clunk_lock.enqueue(append(1)) - clunk_lock.enqueue(append(2)) - clunk_lock.enqueue(append(3)) - - await clunk_lock.run() - await clunk_lock.wait() - - assert data == [1, 3] - - clunk_lock.kill() - - -async def test_post_queue_instant(clunk_lock: ClunkLock): - data = [] - - async def append(number): - data.append(number) - - await clunk_lock.run() - - clunk_lock.enqueue(append(1)) - clunk_lock.enqueue(append(2)) - clunk_lock.enqueue(append(3)) - - await clunk_lock.wait() - assert data == [1, 3] - clunk_lock.kill() - - -async def test_post_queue_with_sleep(clunk_lock: ClunkLock): - data = [] - - async def append(number): - data.append(number) - await asyncio.sleep(0.01) - - await clunk_lock.run() - - clunk_lock.enqueue(append(1)) - clunk_lock.enqueue(append(2)) - clunk_lock.enqueue(append(3)) - - await clunk_lock.wait() - assert data == [1, 3] - clunk_lock.kill() - - -async def test_clunk_acquisition_eviction(clunk: Clunk): - # These should be different due to non-lazy cache eviction - r_1 = clunk.acquire("test") - r_2 = clunk.acquire("test") - assert r_1 is not r_2 - - -async def test_clunk_acquisition_different_keys(clunk: Clunk): - r_1 = clunk.acquire("test") - r_2 = clunk.acquire("test") - assert r_1 is not r_2 - - -async def test_clunk_acquisition_same(clunk: Clunk): - data = [] - - async def task(number): - data.append(number) - await asyncio.sleep(0.05) - - r_1: ClunkLock = clunk.acquire("test") - r_1.enqueue(task(1)) - await r_1.run() - - r_2: ClunkLock = clunk.acquire("test") - assert r_2.is_currently_running is True - assert r_2 is r_1 - - r_2.enqueue(task(1)) - await r_1.wait() - assert data == [1, 1] - - r_2.kill() - await asyncio.sleep(0.01) # Let the loop propagate - assert r_1.is_currently_running is False