From 1e56fc5d06157164135276ca0453bac6aff88985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 16 Sep 2024 22:50:18 +0200 Subject: [PATCH] new: Use valkey-glide instead of redis package --- lacuscore/helpers.py | 10 +- lacuscore/lacus_monitoring.py | 57 ++-- lacuscore/lacuscore.py | 260 +++++++++--------- poetry.lock | 501 ++++++++++++++++++---------------- pyproject.toml | 3 +- 5 files changed, 434 insertions(+), 397 deletions(-) diff --git a/lacuscore/helpers.py b/lacuscore/helpers.py index f438dd7..f9a93d9 100644 --- a/lacuscore/helpers.py +++ b/lacuscore/helpers.py @@ -281,16 +281,18 @@ def check_viewport(cls, viewport: dict[str, int] | None) -> dict[str, int] | Non return viewport raise CaptureSettingsError(f'A viewport must have a width and a height: {viewport}') - def redis_dump(self) -> Mapping[str | bytes, bytes | float | int | str]: - mapping_capture: dict[str | bytes, bytes | float | int | str] = {} + def redis_dump(self) -> Mapping[str | bytes, bytes | str]: + mapping_capture: dict[str | bytes, bytes | str] = {} for key, value in dict(self).items(): if value is None: continue if isinstance(value, bool): - mapping_capture[key] = 1 if value else 0 + mapping_capture[key] = str(1) if value else str(0) elif isinstance(value, (list, dict)): if value: mapping_capture[key] = json.dumps(value) - elif isinstance(value, (bytes, float, int, str)) and value not in ['', b'']: # we're ok with 0 for example + elif isinstance(value, bytes) and value != b'': mapping_capture[key] = value + elif isinstance(value, (float, int, str)) and value != '': # we're ok with 0 for example + mapping_capture[key] = str(value) return mapping_capture diff --git a/lacuscore/lacus_monitoring.py b/lacuscore/lacus_monitoring.py index 8bde869..804cb2f 100644 --- a/lacuscore/lacus_monitoring.py +++ b/lacuscore/lacus_monitoring.py @@ -6,33 +6,38 @@ from datetime import datetime, date -from redis import Redis +from glide import GlideClient, InfBound, RangeByScore, ScoreBoundary class LacusCoreMonitoring(): - def __init__(self, redis_connector: Redis[str]): + def __init__(self, redis_connector: GlideClient): self.redis = redis_connector - def check_redis_up(self) -> bool: - return bool(self.redis.ping()) + async def check_redis_up(self) -> bool: + return bool(await self.redis.ping()) - def get_ongoing_captures(self) -> list[tuple[str, datetime]]: - return [(uuid, datetime.fromtimestamp(timestamp)) for uuid, timestamp in self.redis.zrevrangebyscore('lacus:ongoing', '+Inf', 0, withscores=True)] + async def get_ongoing_captures(self) -> list[tuple[str, datetime]]: + ongoing = await self.redis.zrange_withscores('lacus:ongoing', RangeByScore(InfBound.POS_INF, ScoreBoundary(0))) + return [(uuid.decode(), datetime.fromtimestamp(timestamp)) for uuid, timestamp in ongoing.items()] - def get_capture_settings(self, uuid: str) -> dict[str, str]: - return self.redis.hgetall(f'lacus:capture_settings:{uuid}') + async def get_capture_settings(self, uuid: str) -> dict[str, str]: + captures = await self.redis.hgetall(f'lacus:capture_settings:{uuid}') + return {k.decode(): v.decode() for k, v in captures.items()} - def get_enqueued_captures(self) -> list[tuple[str, float]]: - return self.redis.zrevrangebyscore('lacus:to_capture', '+Inf', '-Inf', withscores=True) + async def get_enqueued_captures(self) -> list[tuple[str, float]]: + enqueued = await self.redis.zrange_withscores('lacus:to_capture', RangeByScore(InfBound.POS_INF, InfBound.NEG_INF)) + return [(uuid.decode(), timestamp) for uuid, timestamp in enqueued.items()] - def get_capture_result(self, uuid: str) -> str | None: - return self.redis.get(f'lacus:capture_results:{uuid}') + async def get_capture_result(self, uuid: str) -> str | None: + if result := await self.redis.get(f'lacus:capture_results:{uuid}'): + return result.decode() + return None - def get_capture_result_size(self, uuid: str) -> str | None: - return self.redis.memory_usage(f'lacus:capture_results:{uuid}') + # async def get_capture_result_size(self, uuid: str) -> str | None: + # return await self.redis.memory_usage(f'lacus:capture_results:{uuid}') - def get_stats(self, d: datetime | date | str | None=None, /, *, cardinality_only: bool=False) -> dict[str, Any]: + async def get_stats(self, d: datetime | date | str | None=None, /, *, cardinality_only: bool=False) -> dict[str, Any]: if d is None: _date = date.today().isoformat() elif isinstance(d, str): @@ -44,20 +49,20 @@ def get_stats(self, d: datetime | date | str | None=None, /, *, cardinality_only else: raise Exception('Invalid type for date ({type(d)})') to_return: dict[str, list[tuple[str, float]] | int | set[str]] = {} - if errors := self.redis.zrevrangebyscore(f'stats:{_date}:errors', '+Inf', 0, withscores=True): - to_return['errors'] = errors + if errors := await self.redis.zrange_withscores(f'stats:{_date}:errors', RangeByScore(InfBound.POS_INF, ScoreBoundary(0))): + to_return['errors'] = [(uuid.decode(), timestamp) for uuid, timestamp in errors.items()] if cardinality_only: - if retry_failed := self.redis.scard(f'stats:{_date}:retry_failed'): + if retry_failed := await self.redis.scard(f'stats:{_date}:retry_failed'): to_return['retry_failed'] = retry_failed - if retry_success := self.redis.scard(f'stats:{_date}:retry_success'): + if retry_success := await self.redis.scard(f'stats:{_date}:retry_success'): to_return['retry_success'] = retry_success - if captures := self.redis.scard(f'stats:{_date}:captures'): + if captures := await self.redis.scard(f'stats:{_date}:captures'): to_return['captures'] = captures else: - if retry_failed_list := self.redis.smembers(f'stats:{_date}:retry_failed'): - to_return['retry_failed'] = retry_failed_list - if retry_success_list := self.redis.smembers(f'stats:{_date}:retry_success'): - to_return['retry_success'] = retry_success_list - if captures_list := self.redis.smembers(f'stats:{_date}:captures'): - to_return['captures'] = captures_list + if retry_failed_list := await self.redis.smembers(f'stats:{_date}:retry_failed'): + to_return['retry_failed'] = {error.decode() for error in retry_failed_list} + if retry_success_list := await self.redis.smembers(f'stats:{_date}:retry_success'): + to_return['retry_success'] = {success.decode() for success in retry_success_list} + if captures_list := await self.redis.smembers(f'stats:{_date}:captures'): + to_return['captures'] = {capture.decode() for capture in captures_list} return to_return diff --git a/lacuscore/lacuscore.py b/lacuscore/lacuscore.py index d33fc44..f6790f5 100644 --- a/lacuscore/lacuscore.py +++ b/lacuscore/lacuscore.py @@ -19,7 +19,7 @@ from datetime import date, timedelta from ipaddress import ip_address, IPv4Address, IPv6Address from tempfile import NamedTemporaryFile -from typing import Literal, Any, overload, cast, Iterator +from typing import Literal, Any, overload, cast, AsyncIterator from uuid import uuid4 from urllib.parse import urlsplit @@ -30,9 +30,7 @@ from playwrightcapture import Capture, PlaywrightCaptureException, InvalidPlaywrightParameter from pydantic import ValidationError -from redis import Redis -from redis.exceptions import ConnectionError as RedisConnectionError -from redis.exceptions import DataError +from glide import GlideClient, Transaction, ConditionalChange, ExpirySet, ExpiryType from ua_parser import user_agent_parser # type: ignore[import-untyped] from . import task_logger @@ -86,7 +84,7 @@ class LacusCore(): :param max_retries: How many times should we re-try a capture if it failed. """ - def __init__(self, redis_connector: Redis[bytes], /, *, + def __init__(self, redis_connector: GlideClient, /, *, max_capture_time: int=3600, tor_proxy: str | None=None, only_global_lookups: bool=True, @@ -107,68 +105,68 @@ def __init__(self, redis_connector: Redis[bytes], /, *, # Enable new chromium headless by default. os.environ["PLAYWRIGHT_CHROMIUM_USE_HEADLESS_NEW"] = "1" - def check_redis_up(self) -> bool: + async def check_redis_up(self) -> bool: """Check if redis is reachable""" - return bool(self.redis.ping()) + return bool(await self.redis.ping()) @overload - def enqueue(self, *, settings: dict[str, Any] | None=None) -> str: + async def enqueue(self, *, settings: dict[str, Any] | None=None) -> str: ... @overload - def enqueue(self, *, - url: str | None=None, - document_name: str | None=None, document: str | None=None, - depth: int=0, - browser: BROWSER | None=None, device_name: str | None=None, - user_agent: str | None=None, - proxy: str | dict[str, str] | None=None, - general_timeout_in_sec: int | None=None, - cookies: list[dict[str, Any]] | None=None, - headers: dict[str, str] | None=None, - http_credentials: dict[str, str] | None=None, - geolocation: dict[str, float] | None=None, - timezone_id: str | None=None, - locale: str | None=None, - color_scheme: str | None=None, - viewport: dict[str, int] | None=None, - referer: str | None=None, - rendered_hostname_only: bool=True, - with_favicon: bool=False, - allow_tracking: bool=False, - force: bool=False, - recapture_interval: int=300, - priority: int=0, - uuid: str | None=None - ) -> str: + async def enqueue(self, *, + url: str | None=None, + document_name: str | None=None, document: str | None=None, + depth: int=0, + browser: BROWSER | None=None, device_name: str | None=None, + user_agent: str | None=None, + proxy: str | dict[str, str] | None=None, + general_timeout_in_sec: int | None=None, + cookies: list[dict[str, Any]] | None=None, + headers: dict[str, str] | None=None, + http_credentials: dict[str, str] | None=None, + geolocation: dict[str, float] | None=None, + timezone_id: str | None=None, + locale: str | None=None, + color_scheme: str | None=None, + viewport: dict[str, int] | None=None, + referer: str | None=None, + rendered_hostname_only: bool=True, + with_favicon: bool=False, + allow_tracking: bool=False, + force: bool=False, + recapture_interval: int=300, + priority: int=0, + uuid: str | None=None + ) -> str: ... - def enqueue(self, *, - settings: dict[str, Any] | None=None, - url: str | None=None, - document_name: str | None=None, document: str | None=None, - depth: int=0, - browser: BROWSER | None=None, device_name: str | None=None, - user_agent: str | None=None, - proxy: str | dict[str, str] | None=None, - general_timeout_in_sec: int | None=None, - cookies: list[dict[str, Any]] | None=None, - headers: dict[str, str] | None=None, - http_credentials: dict[str, str] | None=None, - geolocation: dict[str, float] | None=None, - timezone_id: str | None=None, - locale: str | None=None, - color_scheme: str | None=None, - viewport: dict[str, int] | None=None, - referer: str | None=None, - rendered_hostname_only: bool=True, - with_favicon: bool=False, - allow_tracking: bool=False, - force: bool=False, - recapture_interval: int=300, - priority: int=0, - uuid: str | None=None - ) -> str: + async def enqueue(self, *, + settings: dict[str, Any] | None=None, + url: str | None=None, + document_name: str | None=None, document: str | None=None, + depth: int=0, + browser: BROWSER | None=None, device_name: str | None=None, + user_agent: str | None=None, + proxy: str | dict[str, str] | None=None, + general_timeout_in_sec: int | None=None, + cookies: list[dict[str, Any]] | None=None, + headers: dict[str, str] | None=None, + http_credentials: dict[str, str] | None=None, + geolocation: dict[str, float] | None=None, + timezone_id: str | None=None, + locale: str | None=None, + color_scheme: str | None=None, + viewport: dict[str, int] | None=None, + referer: str | None=None, + rendered_hostname_only: bool=True, + with_favicon: bool=False, + allow_tracking: bool=False, + force: bool=False, + recapture_interval: int=300, + priority: int=0, + uuid: str | None=None + ) -> str: """Enqueue settings. :param settings: Settings as a dictionary @@ -222,14 +220,14 @@ def enqueue(self, *, hash_query = hashlib.sha512(pickle.dumps(to_enqueue)).hexdigest() if not force: - if (existing_uuid := self.redis.get(f'lacus:query_hash:{hash_query}')): + if (existing_uuid := await self.redis.get(f'lacus:query_hash:{hash_query}')): if isinstance(existing_uuid, bytes): return existing_uuid.decode() return existing_uuid if uuid: # Make sure we do not already have a capture with that UUID - if self.get_capture_status(uuid) == CaptureStatus.UNKNOWN: + if await self.get_capture_status(uuid) == CaptureStatus.UNKNOWN: perma_uuid = uuid else: perma_uuid = str(uuid4()) @@ -237,14 +235,17 @@ def enqueue(self, *, else: perma_uuid = str(uuid4()) - p = self.redis.pipeline() - p.set(f'lacus:query_hash:{hash_query}', perma_uuid, nx=True, ex=recapture_interval) - p.hset(f'lacus:capture_settings:{perma_uuid}', mapping=to_enqueue.redis_dump()) + _expire = ExpirySet(ExpiryType.SEC, recapture_interval) + p = Transaction() + p.set(f'lacus:query_hash:{hash_query}', perma_uuid, + conditional_set=ConditionalChange.ONLY_IF_DOES_NOT_EXIST, + expiry=_expire) + p.hset(f'lacus:capture_settings:{perma_uuid}', to_enqueue.redis_dump()) p.zadd('lacus:to_capture', {perma_uuid: priority if priority is not None else 0}) try: - p.execute() - except DataError: - self.master_logger.exception(f'Unable to enqueue: {to_enqueue}') + await self.redis.exec(p) + except Exception as e: + self.master_logger.exception(f'Unable to enqueue ({e}): {to_enqueue}') raise CaptureSettingsError(f'Unable to enqueue: {to_enqueue}') return perma_uuid @@ -264,14 +265,14 @@ def _encode_response(self, capture: CaptureResponse) -> CaptureResponseJson: return encoded_capture @overload - def get_capture(self, uuid: str, *, decode: Literal[True]=True) -> CaptureResponse: + async def get_capture(self, uuid: str, *, decode: Literal[True]=True) -> CaptureResponse: ... @overload - def get_capture(self, uuid: str, *, decode: Literal[False]) -> CaptureResponseJson: + async def get_capture(self, uuid: str, *, decode: Literal[False]) -> CaptureResponseJson: ... - def get_capture(self, uuid: str, *, decode: bool=False) -> CaptureResponse | CaptureResponseJson: + async def get_capture(self, uuid: str, *, decode: bool=False) -> CaptureResponse | CaptureResponseJson: """Get the results of a capture, in a json compatible format or not :param uuid: The UUID if the capture (given by enqueue) @@ -280,11 +281,11 @@ def get_capture(self, uuid: str, *, decode: bool=False) -> CaptureResponse | Cap :return: The capture, decoded or not. """ to_return: CaptureResponse = {'status': CaptureStatus.UNKNOWN} - if self.redis.zscore('lacus:to_capture', uuid): + if await self.redis.zscore('lacus:to_capture', uuid): to_return['status'] = CaptureStatus.QUEUED - elif self.redis.zscore('lacus:ongoing', uuid) is not None: + elif await self.redis.zscore('lacus:ongoing', uuid) is not None: to_return['status'] = CaptureStatus.ONGOING - elif response := self._get_capture_response(uuid): + elif response := await self._get_capture_response(uuid): to_return['status'] = CaptureStatus.DONE to_return.update(response) if decode: @@ -292,48 +293,45 @@ def get_capture(self, uuid: str, *, decode: bool=False) -> CaptureResponse | Cap return self._encode_response(to_return) return to_return - def get_capture_status(self, uuid: str) -> CaptureStatus: + async def get_capture_status(self, uuid: str) -> CaptureStatus: """Get the status of a capture :param uuid: The UUID if the capture (given by enqueue) :return: The status """ - if self.redis.zscore('lacus:to_capture', uuid) is not None: + if await self.redis.zscore('lacus:to_capture', uuid) is not None: return CaptureStatus.QUEUED - elif self.redis.zscore('lacus:ongoing', uuid) is not None: + elif await self.redis.zscore('lacus:ongoing', uuid) is not None: return CaptureStatus.ONGOING - elif self.redis.exists(f'lacus:capture_settings:{uuid}'): + elif await self.redis.exists([f'lacus:capture_settings:{uuid}']): # we might have popped the UUID out of lacus:to_capture # but not pused it in lacus:ongoing yet return CaptureStatus.QUEUED - elif self.redis.exists(f'lacus:capture_results_hash:{uuid}'): + elif await self.redis.exists([f'lacus:capture_results_hash:{uuid}']): return CaptureStatus.DONE - elif self.redis.exists(f'lacus:capture_results:{uuid}'): + elif await self.redis.exists([f'lacus:capture_results:{uuid}']): # TODO: remove in 1.8.* - old format used last in 1.6, and kept no more than 10H in redis return CaptureStatus.DONE return CaptureStatus.UNKNOWN - def consume_queue(self, max_consume: int) -> Iterator[Task]: # type: ignore[type-arg] + async def consume_queue(self, max_consume: int) -> AsyncIterator[Task[None]]: """Trigger the capture for captures with the highest priority. Up to max_consume. :yield: Captures. """ - value: list[tuple[bytes, float]] while max_consume > 0: - value = self.redis.zpopmax('lacus:to_capture') + value = await self.redis.zpopmax('lacus:to_capture') if not value: # Nothing to capture break - if not value[0]: - continue - max_consume -= 1 - uuid: str = value[0][0].decode() - priority: int = int(value[0][1]) - logger = LacusCoreLogAdapter(self.master_logger, {'uuid': uuid}) - yield task_logger.create_task(self._capture(uuid, priority), name=uuid, - logger=logger, - message='Capture raised an uncaught exception') + for _uuid, priority in value.items(): + max_consume -= 1 + uuid = _uuid.decode() + logger = LacusCoreLogAdapter(self.master_logger, {'uuid': uuid}) + yield task_logger.create_task(self._capture(uuid, int(priority)), name=uuid, + logger=logger, + message='Capture raised an uncaught exception') async def _capture(self, uuid: str, priority: int) -> None: """Trigger a specific capture @@ -341,14 +339,14 @@ async def _capture(self, uuid: str, priority: int) -> None: :param uuid: The UUID if the capture (given by enqueue) :param priority: Only for internal use, will decide on the priority of the capture if the try now fails. """ - if self.redis.zscore('lacus:ongoing', uuid) is not None: + if await self.redis.zscore('lacus:ongoing', uuid) is not None: # the capture is already ongoing await asyncio.sleep(1) return logger = LacusCoreLogAdapter(self.master_logger, {'uuid': uuid}) - self.redis.zadd('lacus:ongoing', {uuid: time.time()}) - stats_pipeline = self.redis.pipeline() + await self.redis.zadd('lacus:ongoing', {uuid: time.time()}) + stats_pipeline = Transaction() today = date.today().isoformat() retry = False @@ -356,7 +354,7 @@ async def _capture(self, uuid: str, priority: int) -> None: result: CaptureResponse = {} _to_capture: dict[bytes, Any] = {} url: str = '' - _to_capture = self.redis.hgetall(f'lacus:capture_settings:{uuid}') + _to_capture = await self.redis.hgetall(f'lacus:capture_settings:{uuid}') if not _to_capture: result = {'error': f'No capture settings for {uuid}'} @@ -468,7 +466,7 @@ async def _capture(self, uuid: str, priority: int) -> None: try: logger.debug(f'Capturing {url}') - stats_pipeline.sadd(f'stats:{today}:captures', url) + stats_pipeline.sadd(f'stats:{today}:captures', [url]) async with Capture( browser=browser_engine, device_name=to_capture.device_name, @@ -536,29 +534,28 @@ async def _capture(self, uuid: str, priority: int) -> None: # PlaywrightCapture considers this capture elligible for a retry logger.info('PlaywrightCapture considers it elligible for a retry.') raise RetryCapture('PlaywrightCapture considers it elligible for a retry.') - elif self.redis.exists(f'lacus:capture_retry:{uuid}'): + elif await self.redis.exists([f'lacus:capture_retry:{uuid}']): # this is a retry that worked - stats_pipeline.sadd(f'stats:{today}:retry_success', url) + stats_pipeline.sadd(f'stats:{today}:retry_success', [url]) except RetryCapture: # Check if we already re-tried this capture - _current_retry = self.redis.get(f'lacus:capture_retry:{uuid}') + _current_retry = await self.redis.get(f'lacus:capture_retry:{uuid}') if _current_retry is None: # No retry yet logger.debug(f'Retrying {url} for the first time.') retry = True - self.redis.setex(f'lacus:capture_retry:{uuid}', - self.max_capture_time * (self.max_retries + 10), - self.max_retries) + _expire = ExpirySet(ExpiryType.SEC, self.max_capture_time * (self.max_retries + 10)) + await self.redis.set(f'lacus:capture_retry:{uuid}', str(self.max_retries), expiry=_expire) else: current_retry = int(_current_retry.decode()) if current_retry > 0: logger.debug(f'Retrying {url} for the {self.max_retries - current_retry + 1}th time.') - self.redis.decr(f'lacus:capture_retry:{uuid}') + await self.redis.decr(f'lacus:capture_retry:{uuid}') retry = True else: error_msg = result['error'] if result.get('error') else 'Unknown error' logger.info(f'Retried too many times {url}: {error_msg}') - stats_pipeline.sadd(f'stats:{today}:retry_failed', url) + stats_pipeline.sadd(f'stats:{today}:retry_failed', [url]) except CaptureError: if not result: result = {'error': "No result key, shouldn't happen"} @@ -572,7 +569,7 @@ async def _capture(self, uuid: str, priority: int) -> None: result = {'error': msg} logger.exception(msg) else: - if start_time := self.redis.zscore('lacus:ongoing', uuid): + if start_time := await self.redis.zscore('lacus:ongoing', uuid): runtime = time.time() - start_time logger.info(f'Capture of {url} finished - Runtime: {runtime}s') result['runtime'] = runtime @@ -587,44 +584,44 @@ async def _capture(self, uuid: str, priority: int) -> None: os.unlink(tmp_f.name) if retry: - if self.redis.zcard('lacus:to_capture') == 0: + if await self.redis.zcard('lacus:to_capture') == 0: # Just wait a little bit before retrying await asyncio.sleep(random.randint(5, 10)) - p = self.redis.pipeline() - p.zrem('lacus:ongoing', uuid) + p = Transaction() + p.zrem('lacus:ongoing', [uuid]) p.zadd('lacus:to_capture', {uuid: priority - 1}) - p.execute() + await self.redis.exec(p) else: retry_redis_error = 3 while retry_redis_error > 0: try: - p = self.redis.pipeline() + p = Transaction() if result: self._store_capture_response(p, uuid, result) else: logger.warning('Got no result at all for the capture.') - p.delete(f'lacus:capture_settings:{uuid}') - p.zrem('lacus:ongoing', uuid) - p.execute() + p.delete([f'lacus:capture_settings:{uuid}']) + p.zrem('lacus:ongoing', [uuid]) + await self.redis.exec(p) break - except RedisConnectionError as e: + except Exception as e: logger.warning(f'Unable to store capture result - Redis Connection Error: {e}') retry_redis_error -= 1 await asyncio.sleep(random.randint(5, 10)) else: - self.redis.zrem('lacus:ongoing', uuid) + await self.redis.zrem('lacus:ongoing', [uuid]) stats_pipeline.zincrby(f'stats:{today}:errors', 1, 'Redis Connection') logger.critical('Unable to connect to redis and to push the result of the capture.') # Expire stats in 10 days - expire_time = timedelta(days=10) + expire_time = int(timedelta(days=10).total_seconds()) stats_pipeline.expire(f'stats:{today}:errors', expire_time) stats_pipeline.expire(f'stats:{today}:retry_failed', expire_time) stats_pipeline.expire(f'stats:{today}:retry_success', expire_time) stats_pipeline.expire(f'stats:{today}:captures', expire_time) - stats_pipeline.execute() + await self.redis.exec(stats_pipeline) - def _store_capture_response(self, pipeline: Redis, capture_uuid: str, results: CaptureResponse, # type: ignore[type-arg] + def _store_capture_response(self, pipeline: Transaction, capture_uuid: str, results: CaptureResponse, root_key: str | None=None) -> None: logger = LacusCoreLogAdapter(self.master_logger, {'uuid': capture_uuid}) if root_key is None: @@ -660,19 +657,19 @@ def _store_capture_response(self, pipeline: Redis, capture_uuid: str, results: C hash_to_set[key] = results[key] # type: ignore[literal-required] if hash_to_set: - pipeline.hset(root_key, mapping=hash_to_set) # type: ignore[arg-type] + pipeline.hset(root_key, hash_to_set) # type: ignore[arg-type] # Make sure the key expires pipeline.expire(root_key, 36000) else: logger.critical(f'Nothing to store (Hash: {hash_to_set}) for {root_key}') - def _get_capture_response(self, capture_uuid: str, root_key: str | None=None) -> CaptureResponse | None: + async def _get_capture_response(self, capture_uuid: str, root_key: str | None=None) -> CaptureResponse | None: logger = LacusCoreLogAdapter(self.master_logger, {'uuid': capture_uuid}) if root_key is None: root_key = f'lacus:capture_results_hash:{capture_uuid}' - if not self.redis.exists(root_key): - if old_response := self.redis.get(f'lacus:capture_results:{capture_uuid}'): + if not await self.redis.exists([root_key]): + if old_response := await self.redis.get(f'lacus:capture_results:{capture_uuid}'): # TODO: remove in 1.8.* - old format used last in 1.6, and kept no more than 10H in redis return pickle.loads(zlib.decompress(old_response)) return None @@ -680,7 +677,8 @@ def _get_capture_response(self, capture_uuid: str, root_key: str | None=None) -> # New format and capture done to_return: CaptureResponse = {} - for key, value in self.redis.hgetall(root_key).items(): + results = await self.redis.hgetall(root_key) + for key, value in results.items(): if key == b'har': to_return['har'] = pickle.loads(value) elif key == b'cookies': @@ -690,7 +688,7 @@ def _get_capture_response(self, capture_uuid: str, root_key: str | None=None) -> elif key == b'children': to_return['children'] = [] for child_root_key in sorted(pickle.loads(value)): - if child := self._get_capture_response(capture_uuid, child_root_key): + if child := await self._get_capture_response(capture_uuid, child_root_key): to_return['children'].append(child) # type: ignore[union-attr] elif key in [b'status']: # The value in an int @@ -708,13 +706,13 @@ def _get_capture_response(self, capture_uuid: str, root_key: str | None=None) -> logger.critical(f'Unexpected key in response: {key.decode()} - {value.decode()}') return to_return - def clear_capture(self, uuid: str, reason: str) -> None: + async def clear_capture(self, uuid: str, reason: str) -> None: '''Remove a capture from the list, shouldn't happen unless it is in error''' logger = LacusCoreLogAdapter(self.master_logger, {'uuid': uuid}) - capture_status = self.get_capture_status(uuid) + capture_status = await self.get_capture_status(uuid) if capture_status == CaptureStatus.ONGOING: # Check when it was started. - if start_time := self.redis.zscore('lacus:ongoing', uuid): + if start_time := await self.redis.zscore('lacus:ongoing', uuid): if start_time > time.time() - self.max_capture_time * 1.1: # The capture started recently, wait before clearing it. logger.warning('The capture is (probably) still going, not clearing.') @@ -724,11 +722,11 @@ def clear_capture(self, uuid: str, reason: str) -> None: return logger.warning(f'Clearing capture: {reason}') result: CaptureResponse = {'error': reason} - p = self.redis.pipeline() + p = Transaction() self._store_capture_response(p, uuid, result) - p.delete(f'lacus:capture_settings:{uuid}') - p.zrem('lacus:ongoing', uuid) - p.execute() + p.delete([f'lacus:capture_settings:{uuid}']) + p.zrem('lacus:ongoing', [uuid]) + await self.redis.exec(p) async def __get_ips(self, logger: LacusCoreLogAdapter, hostname: str) -> list[IPv4Address | IPv6Address]: # We need to use dnspython for resolving because socket.getaddrinfo will sometimes be stuck for ~10s diff --git a/poetry.lock b/poetry.lock index 84f778a..09b8b6c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -485,6 +485,17 @@ files = [ [package.dependencies] cffi = ">=1.0.0" +[[package]] +name = "cachetools" +version = "5.5.0" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"}, + {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, +] + [[package]] name = "certifi" version = "2024.8.30" @@ -685,55 +696,6 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -[[package]] -name = "cryptography" -version = "43.0.1" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -optional = false -python-versions = ">=3.7" -files = [ - {file = "cryptography-43.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8385d98f6a3bf8bb2d65a73e17ed87a3ba84f6991c155691c51112075f9ffc5d"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27e613d7077ac613e399270253259d9d53872aaf657471473ebfc9a52935c062"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68aaecc4178e90719e95298515979814bda0cbada1256a4485414860bd7ab962"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:de41fd81a41e53267cb020bb3a7212861da53a7d39f863585d13ea11049cf277"}, - {file = "cryptography-43.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f98bf604c82c416bc829e490c700ca1553eafdf2912a91e23a79d97d9801372a"}, - {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61ec41068b7b74268fa86e3e9e12b9f0c21fcf65434571dbb13d954bceb08042"}, - {file = "cryptography-43.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:014f58110f53237ace6a408b5beb6c427b64e084eb451ef25a28308270086494"}, - {file = "cryptography-43.0.1-cp37-abi3-win32.whl", hash = "sha256:2bd51274dcd59f09dd952afb696bf9c61a7a49dfc764c04dd33ef7a6b502a1e2"}, - {file = "cryptography-43.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:666ae11966643886c2987b3b721899d250855718d6d9ce41b521252a17985f4d"}, - {file = "cryptography-43.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:ac119bb76b9faa00f48128b7f5679e1d8d437365c5d26f1c2c3f0da4ce1b553d"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bbcce1a551e262dfbafb6e6252f1ae36a248e615ca44ba302df077a846a8806"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d4e9129985185a06d849aa6df265bdd5a74ca6e1b736a77959b498e0505b85"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d03a475165f3134f773d1388aeb19c2d25ba88b6a9733c5c590b9ff7bbfa2e0c"}, - {file = "cryptography-43.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:511f4273808ab590912a93ddb4e3914dfd8a388fed883361b02dea3791f292e1"}, - {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:80eda8b3e173f0f247f711eef62be51b599b5d425c429b5d4ca6a05e9e856baa"}, - {file = "cryptography-43.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38926c50cff6f533f8a2dae3d7f19541432610d114a70808f0926d5aaa7121e4"}, - {file = "cryptography-43.0.1-cp39-abi3-win32.whl", hash = "sha256:a575913fb06e05e6b4b814d7f7468c2c660e8bb16d8d5a1faf9b33ccc569dd47"}, - {file = "cryptography-43.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:d75601ad10b059ec832e78823b348bfa1a59f6b8d545db3a24fd44362a1564cb"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ea25acb556320250756e53f9e20a4177515f012c9eaea17eb7587a8c4d8ae034"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c1332724be35d23a854994ff0b66530119500b6053d0bd3363265f7e5e77288d"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1007b3ef89946dbbb515aeeb41e30203b004f0b4b00e5e16078b518563289"}, - {file = "cryptography-43.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5b43d1ea6b378b54a1dc99dd8a2b5be47658fe9a7ce0a58ff0b55f4b43ef2b84"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:88cce104c36870d70c49c7c8fd22885875d950d9ee6ab54df2745f83ba0dc365"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:9d3cdb25fa98afdd3d0892d132b8d7139e2c087da1712041f6b762e4f807cc96"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e710bf40870f4db63c3d7d929aa9e09e4e7ee219e703f949ec4073b4294f6172"}, - {file = "cryptography-43.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7c05650fe8023c5ed0d46793d4b7d7e6cd9c04e68eabe5b0aeea836e37bdcec2"}, - {file = "cryptography-43.0.1.tar.gz", hash = "sha256:203e92a75716d8cfb491dc47c79e17d0d9207ccffcbcb35f598fbe463ae3444d"}, -] - -[package.dependencies] -cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} - -[package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] -docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] -nox = ["nox"] -pep8test = ["check-sdist", "click", "mypy", "ruff"] -sdist = ["build"] -ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi", "cryptography-vectors (==43.0.1)", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] -test-randomorder = ["pytest-randomly"] - [[package]] name = "dateparser" version = "1.2.0" @@ -947,6 +909,102 @@ files = [ {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"}, ] +[[package]] +name = "google-api-core" +version = "2.19.2" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google_api_core-2.19.2-py3-none-any.whl", hash = "sha256:53ec0258f2837dd53bbd3d3df50f5359281b3cc13f800c941dd15a9b5a415af4"}, + {file = "google_api_core-2.19.2.tar.gz", hash = "sha256:ca07de7e8aa1c98a8bfca9321890ad2340ef7f2eb136e558cee68f24b94b0a8f"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +proto-plus = ">=1.22.3,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-api-python-client" +version = "2.85.0" +description = "Google API Client Library for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-python-client-2.85.0.tar.gz", hash = "sha256:07b21ef21a542dd69cd7c09817a6079b2769cc2a791981402e8f0fcdb2d47f90"}, + {file = "google_api_python_client-2.85.0-py2.py3-none-any.whl", hash = "sha256:baf3c6f9b1679d89fcb88c29941a8b04b9a815d721880786baecc6a7f5bd376f"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.19.0,<3.0.0dev" +google-auth-httplib2 = ">=0.1.0" +httplib2 = ">=0.15.0,<1dev" +uritemplate = ">=3.0.1,<5" + +[[package]] +name = "google-auth" +version = "2.34.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google_auth-2.34.0-py2.py3-none-any.whl", hash = "sha256:72fd4733b80b6d777dcde515628a9eb4a577339437012874ea286bca7261ee65"}, + {file = "google_auth-2.34.0.tar.gz", hash = "sha256:8eb87396435c19b20d32abd2f984e31c191a15284af72eb922f10e5bde9c04cc"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography", "pyopenssl"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-auth-httplib2" +version = "0.2.0" +description = "Google Authentication Library: httplib2 transport" +optional = false +python-versions = "*" +files = [ + {file = "google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05"}, + {file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"}, +] + +[package.dependencies] +google-auth = "*" +httplib2 = ">=0.19.0" + +[[package]] +name = "googleapis-common-protos" +version = "1.65.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis_common_protos-1.65.0-py2.py3-none-any.whl", hash = "sha256:2972e6c496f435b92590fd54045060867f3fe9be2c82ab148fc8885035479a63"}, + {file = "googleapis_common_protos-1.65.0.tar.gz", hash = "sha256:334a29d07cddc3aa01dee4988f9afd9b2916ee2ff49d6b757155dc0d197852c0"}, +] + +[package.dependencies] +protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + [[package]] name = "greenlet" version = "3.0.3" @@ -1019,119 +1077,33 @@ docs = ["Sphinx", "furo"] test = ["objgraph", "psutil"] [[package]] -name = "hiredis" -version = "3.0.0" -description = "Python wrapper for hiredis" +name = "httplib2" +version = "0.22.0" +description = "A comprehensive HTTP client library." optional = false -python-versions = ">=3.8" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ - {file = "hiredis-3.0.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:4b182791c41c5eb1d9ed736f0ff81694b06937ca14b0d4dadde5dadba7ff6dae"}, - {file = "hiredis-3.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:13c275b483a052dd645eb2cb60d6380f1f5215e4c22d6207e17b86be6dd87ffa"}, - {file = "hiredis-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1018cc7f12824506f165027eabb302735b49e63af73eb4d5450c66c88f47026"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83a29cc7b21b746cb6a480189e49f49b2072812c445e66a9e38d2004d496b81c"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e241fab6332e8fb5f14af00a4a9c6aefa22f19a336c069b7ddbf28ef8341e8d6"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1fb8de899f0145d6c4d5d4bd0ee88a78eb980a7ffabd51e9889251b8f58f1785"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b23291951959141173eec10f8573538e9349fa27f47a0c34323d1970bf891ee5"}, - {file = "hiredis-3.0.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e421ac9e4b5efc11705a0d5149e641d4defdc07077f748667f359e60dc904420"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:77c8006c12154c37691b24ff293c077300c22944018c3ff70094a33e10c1d795"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:41afc0d3c18b59eb50970479a9c0e5544fb4b95e3a79cf2fbaece6ddefb926fe"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:04ccae6dcd9647eae6025425ab64edb4d79fde8b9e6e115ebfabc6830170e3b2"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:fe91d62b0594db5ea7d23fc2192182b1a7b6973f628a9b8b2e0a42a2be721ac6"}, - {file = "hiredis-3.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:99516d99316062824a24d145d694f5b0d030c80da693ea6f8c4ecf71a251d8bb"}, - {file = "hiredis-3.0.0-cp310-cp310-win32.whl", hash = "sha256:562eaf820de045eb487afaa37e6293fe7eceb5b25e158b5a1974b7e40bf04543"}, - {file = "hiredis-3.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:a1c81c89ed765198da27412aa21478f30d54ef69bf5e4480089d9c3f77b8f882"}, - {file = "hiredis-3.0.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:4664dedcd5933364756d7251a7ea86d60246ccf73a2e00912872dacbfcef8978"}, - {file = "hiredis-3.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:47de0bbccf4c8a9f99d82d225f7672b9dd690d8fd872007b933ef51a302c9fa6"}, - {file = "hiredis-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e43679eca508ba8240d016d8cca9d27342d70184773c15bea78a23c87a1922f1"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:13c345e7278c210317e77e1934b27b61394fee0dec2e8bd47e71570900f75823"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00018f22f38530768b73ea86c11f47e8d4df65facd4e562bd78773bd1baef35e"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ea3a86405baa8eb0d3639ced6926ad03e07113de54cb00fd7510cb0db76a89d"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c073848d2b1d5561f3903879ccf4e1a70c9b1e7566c7bdcc98d082fa3e7f0a1d"}, - {file = "hiredis-3.0.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a8dffb5f5b3415a4669d25de48b617fd9d44b0bccfc4c2ab24b06406ecc9ecb"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:22c17c96143c2a62dfd61b13803bc5de2ac526b8768d2141c018b965d0333b66"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c3ece960008dab66c6b8bb3a1350764677ee7c74ccd6270aaf1b1caf9ccebb46"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f75999ae00a920f7dce6ecae76fa5e8674a3110e5a75f12c7a2c75ae1af53396"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e069967cbd5e1900aafc4b5943888f6d34937fc59bf8918a1a546cb729b4b1e4"}, - {file = "hiredis-3.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0aacc0a78e1d94d843a6d191f224a35893e6bdfeb77a4a89264155015c65f126"}, - {file = "hiredis-3.0.0-cp311-cp311-win32.whl", hash = "sha256:719c32147ba29528cb451f037bf837dcdda4ff3ddb6cdb12c4216b0973174718"}, - {file = "hiredis-3.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:bdc144d56333c52c853c31b4e2e52cfbdb22d3da4374c00f5f3d67c42158970f"}, - {file = "hiredis-3.0.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:484025d2eb8f6348f7876fc5a2ee742f568915039fcb31b478fd5c242bb0fe3a"}, - {file = "hiredis-3.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:fcdb552ffd97151dab8e7bc3ab556dfa1512556b48a367db94b5c20253a35ee1"}, - {file = "hiredis-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bb6f9fd92f147ba11d338ef5c68af4fd2908739c09e51f186e1d90958c68cc1"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa86bf9a0ed339ec9e8a9a9d0ae4dccd8671625c83f9f9f2640729b15e07fbfd"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e194a0d5df9456995d8f510eab9f529213e7326af6b94770abf8f8b7952ddcaa"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c8a1df39d74ec507d79c7a82c8063eee60bf80537cdeee652f576059b9cdd15c"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f91456507427ba36fd81b2ca11053a8e112c775325acc74e993201ea912d63e9"}, - {file = "hiredis-3.0.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9862db92ef67a8a02e0d5370f07d380e14577ecb281b79720e0d7a89aedb9ee5"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d10fcd9e0eeab835f492832b2a6edb5940e2f1230155f33006a8dfd3bd2c94e4"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:48727d7d405d03977d01885f317328dc21d639096308de126c2c4e9950cbd3c9"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e0bb6102ebe2efecf8a3292c6660a0e6fac98176af6de67f020bea1c2343717"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:df274e3abb4df40f4c7274dd3e587dfbb25691826c948bc98d5fead019dfb001"}, - {file = "hiredis-3.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:034925b5fb514f7b11aac38cd55b3fd7e9d3af23bd6497f3f20aa5b8ba58e232"}, - {file = "hiredis-3.0.0-cp312-cp312-win32.whl", hash = "sha256:120f2dda469b28d12ccff7c2230225162e174657b49cf4cd119db525414ae281"}, - {file = "hiredis-3.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:e584fe5f4e6681d8762982be055f1534e0170f6308a7a90f58d737bab12ff6a8"}, - {file = "hiredis-3.0.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:122171ff47d96ed8dd4bba6c0e41d8afaba3e8194949f7720431a62aa29d8895"}, - {file = "hiredis-3.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:ba9fc605ac558f0de67463fb588722878641e6fa1dabcda979e8e69ff581d0bd"}, - {file = "hiredis-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a631e2990b8be23178f655cae8ac6c7422af478c420dd54e25f2e26c29e766f1"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63482db3fadebadc1d01ad33afa6045ebe2ea528eb77ccaabd33ee7d9c2bad48"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f669212c390eebfbe03c4e20181f5970b82c5d0a0ad1df1785f7ffbe7d61150"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a49ef161739f8018c69b371528bdb47d7342edfdee9ddc75a4d8caddf45a6e"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98a152052b8878e5e43a2e3a14075218adafc759547c98668a21e9485882696c"}, - {file = "hiredis-3.0.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50a196af0ce657fcde9bf8a0bbe1032e22c64d8fcec2bc926a35e7ff68b3a166"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:f2f312eef8aafc2255e3585dcf94d5da116c43ef837db91db9ecdc1bc930072d"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:6ca41fa40fa019cde42c21add74aadd775e71458051a15a352eabeb12eb4d084"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:6eecb343c70629f5af55a8b3e53264e44fa04e155ef7989de13668a0cb102a90"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:c3fdad75e7837a475900a1d3a5cc09aa024293c3b0605155da2d42f41bc0e482"}, - {file = "hiredis-3.0.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:8854969e7480e8d61ed7549eb232d95082a743e94138d98d7222ba4e9f7ecacd"}, - {file = "hiredis-3.0.0-cp38-cp38-win32.whl", hash = "sha256:f114a6c86edbf17554672b050cce72abf489fe58d583c7921904d5f1c9691605"}, - {file = "hiredis-3.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:7d99b91e42217d7b4b63354b15b41ce960e27d216783e04c4a350224d55842a4"}, - {file = "hiredis-3.0.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:4c6efcbb5687cf8d2aedcc2c3ed4ac6feae90b8547427d417111194873b66b06"}, - {file = "hiredis-3.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5b5cff42a522a0d81c2ae7eae5e56d0ee7365e0c4ad50c4de467d8957aff4414"}, - {file = "hiredis-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:82f794d564f4bc76b80c50b03267fe5d6589e93f08e66b7a2f674faa2fa76ebc"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7a4c1791d7aa7e192f60fe028ae409f18ccdd540f8b1e6aeb0df7816c77e4a4"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2537b2cd98192323fce4244c8edbf11f3cac548a9d633dbbb12b48702f379f4"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fed69bbaa307040c62195a269f82fc3edf46b510a17abb6b30a15d7dab548df"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:869f6d5537d243080f44253491bb30aa1ec3c21754003b3bddeadedeb65842b0"}, - {file = "hiredis-3.0.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d435ae89073d7cd51e6b6bf78369c412216261c9c01662e7008ff00978153729"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:204b79b30a0e6be0dc2301a4d385bb61472809f09c49f400497f1cdd5a165c66"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3ea635101b739c12effd189cc19b2671c268abb03013fd1f6321ca29df3ca625"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:f359175197fd833c8dd7a8c288f1516be45415bb5c939862ab60c2918e1e1943"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ac6d929cb33dd12ad3424b75725975f0a54b5b12dbff95f2a2d660c510aa106d"}, - {file = "hiredis-3.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:100431e04d25a522ef2c3b94f294c4219c4de3bfc7d557b6253296145a144c11"}, - {file = "hiredis-3.0.0-cp39-cp39-win32.whl", hash = "sha256:e1a9c14ae9573d172dc050a6f63a644457df5d01ec4d35a6a0f097f812930f83"}, - {file = "hiredis-3.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:54a6dd7b478e6eb01ce15b3bb5bf771e108c6c148315bf194eb2ab776a3cac4d"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:50da7a9edf371441dfcc56288d790985ee9840d982750580710a9789b8f4a290"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9b285ef6bf1581310b0d5e8f6ce64f790a1c40e89c660e1320b35f7515433672"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dcfa684966f25b335072115de2f920228a3c2caf79d4bfa2b30f6e4f674a948"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a41be8af1fd78ca97bc948d789a09b730d1e7587d07ca53af05758f31f4b985d"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:038756db735e417ab36ee6fd7725ce412385ed2bd0767e8179a4755ea11b804f"}, - {file = "hiredis-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fcecbd39bd42cef905c0b51c9689c39d0cc8b88b1671e7f40d4fb213423aef3a"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:a131377493a59fb0f5eaeb2afd49c6540cafcfba5b0b3752bed707be9e7c4eaf"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d22c53f0ec5c18ecb3d92aa9420563b1c5d657d53f01356114978107b00b860"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8a91e9520fbc65a799943e5c970ffbcd67905744d8becf2e75f9f0a5e8414f0"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3dc8043959b50141df58ab4f398e8ae84c6f9e673a2c9407be65fc789138f4a6"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:51b99cfac514173d7b8abdfe10338193e8a0eccdfe1870b646009d2fb7cbe4b5"}, - {file = "hiredis-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:fa1fcad89d8a41d8dc10b1e54951ec1e161deabd84ed5a2c95c3c7213bdb3514"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:898636a06d9bf575d2c594129085ad6b713414038276a4bfc5db7646b8a5be78"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:466f836dbcf86de3f9692097a7a01533dc9926986022c6617dc364a402b265c5"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23142a8af92a13fc1e3f2ca1d940df3dcf2af1d176be41fe8d89e30a837a0b60"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:793c80a3d6b0b0e8196a2d5de37a08330125668c8012922685e17aa9108c33ac"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:467d28112c7faa29b7db743f40803d927c8591e9da02b6ce3d5fadc170a542a2"}, - {file = "hiredis-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:dc384874a719c767b50a30750f937af18842ee5e288afba95a5a3ed703b1515a"}, - {file = "hiredis-3.0.0.tar.gz", hash = "sha256:fed8581ae26345dea1f1e0d1a96e05041a727a45e7d8d459164583e23c6ac441"}, + {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, + {file = "httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81"}, ] +[package.dependencies] +pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0.2,<3.0.3 || >3.0.3,<4", markers = "python_version > \"3.0\""} + [[package]] name = "idna" -version = "3.8" +version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" files = [ - {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, - {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, ] +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + [[package]] name = "imagesize" version = "1.4.1" @@ -1873,6 +1845,43 @@ files = [ [package.dependencies] wcwidth = "*" +[[package]] +name = "proto-plus" +version = "1.24.0" +description = "Beautiful, Pythonic protocol buffers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "proto-plus-1.24.0.tar.gz", hash = "sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445"}, + {file = "proto_plus-1.24.0-py3-none-any.whl", hash = "sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<6.0.0dev" + +[package.extras] +testing = ["google-api-core (>=1.31.5)"] + +[[package]] +name = "protobuf" +version = "5.28.1" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-5.28.1-cp310-abi3-win32.whl", hash = "sha256:fc063acaf7a3d9ca13146fefb5b42ac94ab943ec6e978f543cd5637da2d57957"}, + {file = "protobuf-5.28.1-cp310-abi3-win_amd64.whl", hash = "sha256:4c7f5cb38c640919791c9f74ea80c5b82314c69a8409ea36f2599617d03989af"}, + {file = "protobuf-5.28.1-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:4304e4fceb823d91699e924a1fdf95cde0e066f3b1c28edb665bda762ecde10f"}, + {file = "protobuf-5.28.1-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:0dfd86d2b5edf03d91ec2a7c15b4e950258150f14f9af5f51c17fa224ee1931f"}, + {file = "protobuf-5.28.1-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:51f09caab818707ab91cf09cc5c156026599cf05a4520779ccbf53c1b352fb25"}, + {file = "protobuf-5.28.1-cp38-cp38-win32.whl", hash = "sha256:1b04bde117a10ff9d906841a89ec326686c48ececeb65690f15b8cabe7149495"}, + {file = "protobuf-5.28.1-cp38-cp38-win_amd64.whl", hash = "sha256:cabfe43044ee319ad6832b2fda332646f9ef1636b0130186a3ae0a52fc264bb4"}, + {file = "protobuf-5.28.1-cp39-cp39-win32.whl", hash = "sha256:4b4b9a0562a35773ff47a3df823177ab71a1f5eb1ff56d8f842b7432ecfd7fd2"}, + {file = "protobuf-5.28.1-cp39-cp39-win_amd64.whl", hash = "sha256:f24e5d70e6af8ee9672ff605d5503491635f63d5db2fffb6472be78ba62efd8f"}, + {file = "protobuf-5.28.1-py3-none-any.whl", hash = "sha256:c529535e5c0effcf417682563719e5d8ac8d2b93de07a56108b4c2d436d7a29a"}, + {file = "protobuf-5.28.1.tar.gz", hash = "sha256:42597e938f83bb7f3e4b35f03aa45208d49ae8d5bcb4bc10b9fc825e0ab5e423"}, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -1909,6 +1918,31 @@ files = [ {file = "puremagic-1.27.tar.gz", hash = "sha256:7cb316f40912f56f34149f8ebdd77a91d099212d2ed936feb2feacfc7cbce2c1"}, ] +[[package]] +name = "pyasn1" +version = "0.6.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, + {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.1" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd"}, + {file = "pyasn1_modules-0.4.1.tar.gz", hash = "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.7.0" + [[package]] name = "pycares" version = "4.4.0" @@ -2152,6 +2186,20 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pyparsing" +version = "3.1.4" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.6.8" +files = [ + {file = "pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c"}, + {file = "pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + [[package]] name = "pytest" version = "8.3.3" @@ -2219,25 +2267,6 @@ files = [ {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, ] -[[package]] -name = "redis" -version = "5.0.8" -description = "Python client for Redis database and key-value store" -optional = false -python-versions = ">=3.7" -files = [ - {file = "redis-5.0.8-py3-none-any.whl", hash = "sha256:56134ee08ea909106090934adc36f65c9bcbbaecea5b21ba704ba6fb561f8eb4"}, - {file = "redis-5.0.8.tar.gz", hash = "sha256:0c5b10d387568dfe0698c6fad6615750c24170e548ca2deac10c649d463e9870"}, -] - -[package.dependencies] -async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} -hiredis = {version = ">1.0.0", optional = true, markers = "extra == \"hiredis\""} - -[package.extras] -hiredis = ["hiredis (>1.0.0)"] -ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] - [[package]] name = "regex" version = "2024.9.11" @@ -2362,15 +2391,29 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + [[package]] name = "setuptools" -version = "74.1.2" +version = "74.1.3" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-74.1.2-py3-none-any.whl", hash = "sha256:5f4c08aa4d3ebcb57a50c33b1b07e94315d7fc7230f7115e47fc99776c8ce308"}, - {file = "setuptools-74.1.2.tar.gz", hash = "sha256:95b40ed940a1c67eb70fc099094bd6e99c6ee7c23aa2306f4d2697ba7916f9c6"}, + {file = "setuptools-74.1.3-py3-none-any.whl", hash = "sha256:1cfd66bfcf197bce344da024c8f5b35acc4dcb7ca5202246a75296b4883f6851"}, + {file = "setuptools-74.1.3.tar.gz", hash = "sha256:fbb126f14b0b9ffa54c4574a50ae60673bbe8ae0b1645889d10b3b14f5891d28"}, ] [package.extras] @@ -2769,20 +2812,6 @@ files = [ [package.dependencies] types-html5lib = "*" -[[package]] -name = "types-cffi" -version = "1.16.0.20240331" -description = "Typing stubs for cffi" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types-cffi-1.16.0.20240331.tar.gz", hash = "sha256:b8b20d23a2b89cfed5f8c5bc53b0cb8677c3aac6d970dbc771e28b9c698f5dee"}, - {file = "types_cffi-1.16.0.20240331-py3-none-any.whl", hash = "sha256:a363e5ea54a4eb6a4a105d800685fde596bc318089b025b27dee09849fe41ff0"}, -] - -[package.dependencies] -types-setuptools = "*" - [[package]] name = "types-html5lib" version = "1.1.11.20240806" @@ -2794,61 +2823,20 @@ files = [ {file = "types_html5lib-1.1.11.20240806-py3-none-any.whl", hash = "sha256:575c4fd84ba8eeeaa8520c7e4c7042b7791f5ec3e9c0a5d5c418124c42d9e7e4"}, ] -[[package]] -name = "types-pyopenssl" -version = "24.1.0.20240722" -description = "Typing stubs for pyOpenSSL" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types-pyOpenSSL-24.1.0.20240722.tar.gz", hash = "sha256:47913b4678a01d879f503a12044468221ed8576263c1540dcb0484ca21b08c39"}, - {file = "types_pyOpenSSL-24.1.0.20240722-py3-none-any.whl", hash = "sha256:6a7a5d2ec042537934cfb4c9d4deb0e16c4c6250b09358df1f083682fe6fda54"}, -] - -[package.dependencies] -cryptography = ">=35.0.0" -types-cffi = "*" - -[[package]] -name = "types-redis" -version = "4.6.0.20240903" -description = "Typing stubs for redis" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types-redis-4.6.0.20240903.tar.gz", hash = "sha256:4bab1a378dbf23c2c95c370dfdb89a8f033957c4fd1a53fee71b529c182fe008"}, - {file = "types_redis-4.6.0.20240903-py3-none-any.whl", hash = "sha256:0e7537e5c085fe96b7d468d5edae0cf667b4ba4b62c6e4a5dfc340bd3b868c23"}, -] - -[package.dependencies] -cryptography = ">=35.0.0" -types-pyOpenSSL = "*" - [[package]] name = "types-requests" -version = "2.32.0.20240907" +version = "2.32.0.20240914" description = "Typing stubs for requests" optional = false python-versions = ">=3.8" files = [ - {file = "types-requests-2.32.0.20240907.tar.gz", hash = "sha256:ff33935f061b5e81ec87997e91050f7b4af4f82027a7a7a9d9aaea04a963fdf8"}, - {file = "types_requests-2.32.0.20240907-py3-none-any.whl", hash = "sha256:1d1e79faeaf9d42def77f3c304893dea17a97cae98168ac69f3cb465516ee8da"}, + {file = "types-requests-2.32.0.20240914.tar.gz", hash = "sha256:2850e178db3919d9bf809e434eef65ba49d0e7e33ac92d588f4a5e295fffd405"}, + {file = "types_requests-2.32.0.20240914-py3-none-any.whl", hash = "sha256:59c2f673eb55f32a99b2894faf6020e1a9f4a402ad0f192bfee0b64469054310"}, ] [package.dependencies] urllib3 = ">=2" -[[package]] -name = "types-setuptools" -version = "74.1.0.20240907" -description = "Typing stubs for setuptools" -optional = false -python-versions = ">=3.8" -files = [ - {file = "types-setuptools-74.1.0.20240907.tar.gz", hash = "sha256:0abdb082552ca966c1e5fc244e4853adc62971f6cd724fb1d8a3713b580e5a65"}, - {file = "types_setuptools-74.1.0.20240907-py3-none-any.whl", hash = "sha256:15b38c8e63ca34f42f6063ff4b1dd662ea20086166d5ad6a102e670a52574120"}, -] - [[package]] name = "typing-extensions" version = "4.12.2" @@ -2900,6 +2888,17 @@ files = [ {file = "ua_parser-0.18.0-py2.py3-none-any.whl", hash = "sha256:9d94ac3a80bcb0166823956a779186c746b50ea4c9fd9bf30fdb758553c38950"}, ] +[[package]] +name = "uritemplate" +version = "4.1.1" +description = "Implementation of RFC 6570 URI Templates" +optional = false +python-versions = ">=3.6" +files = [ + {file = "uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"}, + {file = "uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0"}, +] + [[package]] name = "urllib3" version = "2.2.3" @@ -2917,6 +2916,40 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "valkey-glide" +version = "1.0.1" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "valkey_glide-1.0.1-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:3789e0c8de32039573c11bf55bbc086d28e0cb17a7f5f2d348f95f436e8e2920"}, + {file = "valkey_glide-1.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:163b9c9e69b3d58a7fb99bb233ec67571d0d63a316c8047bbf3e70e9d2c31283"}, + {file = "valkey_glide-1.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4bf512a17b78c5baf535f4dcdf582cc9158d587188bb852162d50019e8a8d4"}, + {file = "valkey_glide-1.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:7c5e874f337218e33a3143ee4479e20178aa3f1e429bd7b85f4f27911b80a286"}, + {file = "valkey_glide-1.0.1-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:dcd3e84018ce6028bf9297ec2946969379b174b4e064cf65d7b81c345c5a6ca2"}, + {file = "valkey_glide-1.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:79d615856402b95824c8fa9e20e2b3ba609d3fd866b0b8757f5dc840f6d56dc6"}, + {file = "valkey_glide-1.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bed3d73bb8bbdb87068798ffb5c055b0e5aabf3698bf794dd83ff5d38c614195"}, + {file = "valkey_glide-1.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:a982e1c6b4c259e30475e36cfd5e7c8ee087a36dfb321749e3f85abcfef5e329"}, + {file = "valkey_glide-1.0.1-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:af6381b4733a7d40c0c1c1230c686ef3796e77537bcc50ab4fab055fc3c399a5"}, + {file = "valkey_glide-1.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bff890ca9b5e0a803a6c41527c812d9ad1bf261bbf674a3f3076959d4b4345ee"}, + {file = "valkey_glide-1.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:904fcef9b5583cde6c9893fce3bbc21e4e86d99b4a5695e9f5ff79a32942dd09"}, + {file = "valkey_glide-1.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:7c0bf9d3fa39f343540636a302e6302e2fd7bd802da8e6f5eac847e17d2d76d4"}, + {file = "valkey_glide-1.0.1-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:15b3aa05c379252545ec87bd0b87b1ec330aaeec70306cd8586f54eb4d38c6c6"}, + {file = "valkey_glide-1.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ff97efe706f334ec817463508205297e0682d4b1f5bcbafda65059c456f77f45"}, + {file = "valkey_glide-1.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e3be2fd99be19cae3e3441921ad809b49b40791aa7226f0c0c75e829a69d1bc"}, + {file = "valkey_glide-1.0.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:6c8c09007b3e4b9e31b8dbedd203b7e733c09f19b564b061559e8b0f74532776"}, + {file = "valkey_glide-1.0.1-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:83e53a62d04a8ad38ea9ca55ad1c0b3862d33692017baa6dc3863f65e225bf0b"}, + {file = "valkey_glide-1.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e9e13b449701fb2234dda8b5d49573d10ad14d61271f2ad253aa10f7972ecd1f"}, + {file = "valkey_glide-1.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fa09e6810ca8ab7e1ecef37eac914d42eee8b835c98c2806d21846af5ba3c5b"}, + {file = "valkey_glide-1.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:770b6bd9b1b07a3de0912fe050bfcd656c66189019e5c230ea983e516cd22368"}, +] + +[package.dependencies] +async-timeout = ">=4.0.2" +google-api-python-client = "2.85.0" +typing-extensions = ">=4.8.0" + [[package]] name = "w3lib" version = "2.2.1" @@ -3046,13 +3079,13 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.20.1" +version = "3.20.2" description = "Backport of pathlib-compatible object wrapper for zip files" optional = true python-versions = ">=3.8" files = [ - {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, - {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, + {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, + {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, ] [package.extras] @@ -3069,4 +3102,4 @@ docs = ["Sphinx", "Sphinx", "Sphinx"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "52bba2f6e75dbbaae7f07041bf62f97368f393cc73be595197260bac0376a84f" +content-hash = "6e6701e5531724b2cfc9216e14fe86a8283736fb47b58b5ecba83a0935790c29" diff --git a/pyproject.toml b/pyproject.toml index 20665d8..bfa8a94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,18 +38,17 @@ Sphinx = [ playwrightcapture = {extras = ["recaptcha"], version = "^1.25.15"} defang = "^0.5.3" ua-parser = "^0.18.0" -redis = {version = "^5.0.8", extras = ["hiredis"]} dnspython = "^2.6.1" async-timeout = {version = "^4.0.3", python = "<3.11"} pydantic = "^2.9.1" eval-type-backport = {version = "^0.2.0", python = "<3.10"} +valkey-glide = "^1.0.1" [tool.poetry.extras] docs = ["Sphinx"] [tool.poetry.group.dev.dependencies] mypy = "^1.11.2" -types-redis = {version = "^4.6.0.20240903"} types-requests = "^2.32.0.20240907" types-beautifulsoup4 = "^4.12.0.20240907" ipython = [