diff --git a/google_custom_search/adapter.py b/google_custom_search/adapter.py index 9aa3c30..3c34c71 100644 --- a/google_custom_search/adapter.py +++ b/google_custom_search/adapter.py @@ -15,6 +15,8 @@ from .errors import AsyncError, ApiNotEnabled from .types import Item +from typing import AsyncGenerator + class BaseAdapter(metaclass=ABCMeta): """This is the base class for adapters. @@ -42,6 +44,9 @@ def request(self, method: str, path: str, *args, **kwargs) -> Any: def search(self, *args, **kwargs) -> List[Item]: ... + async def asearch(self, *_args, **_kwargs) -> AsyncGenerator[Item, None]: + raise NotImplementedError("You can only use 'asearch' on an asynchronous adapter") + def _from_dict(self, data: dict) -> List[Item]: if data.get('error'): raise ApiNotEnabled( @@ -52,9 +57,10 @@ def _from_dict(self, data: dict) -> List[Item]: def _payload_maker( self, query: str, *, safe: bool = False, - filter_: bool = False + filter_: bool = False, + **kwargs ) -> dict: - payload = { + payload = kwargs | { "key": self.apikey, "cx": self.engine_id, "q": query @@ -89,8 +95,8 @@ def search(self, *args, **kwargs) -> List[Item]: class AiohttpAdapter(BaseAdapter): "This class is aiohttpadapter for async mode." - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__(self, apikey, engine_id, *args, **kwargs): + super().__init__(apikey, engine_id) if not async_mode: raise AsyncError( "This adapter use aiohttp, so please install aiohttp") @@ -106,4 +112,26 @@ async def search(self, *args, **kwargs) -> List[Item]: r = await self.request( "GET", "/", params=self._payload_maker(*args, **kwargs) ) - return self._from_dict(await r.json()) + return self._from_dict(r) + + async def asearch(self, *args, **kwargs) -> AsyncGenerator[Item, None]: + limit = kwargs.get("limit", 100) + + if "limit" in kwargs: + del kwargs["limit"] + + while True: + page = await self.search(*args, **kwargs) + + for result in page: + yield result + + kwargs["start"] = kwargs.get("start", 1) + kwargs.get("num", 10) + + if kwargs["start"] + kwargs.get("num", 10) > limit: + kwargs["num"] = limit - kwargs["start"] + 1 # both ends of the range are inclusive + + if kwargs.get("num", 10) <= 0: + return + + diff --git a/google_custom_search/search.py b/google_custom_search/search.py index c699129..6545d29 100644 --- a/google_custom_search/search.py +++ b/google_custom_search/search.py @@ -1,34 +1,38 @@ -# google-custom-seaerch - search - -from typing import List - -from .types import Item -from .adapter import BaseAdapter - - -class CustomSearch: - """This is the class used when using Google Custom Search. - - Args: - adapter (BaseAdapter): Insert adapter - """ - APIURL: str = "https://www.googleapis.com/customsearch/v1" - - def __init__(self, adapter: BaseAdapter): - self.adapter = adapter - - def search(self, *args, **kwargs) -> List[Item]: - """This is searched using api. - - Args: - query (str): Search keyword - safe (bool): Using safe mode - filter_ (filter): Use filter mode - - Returns: - List[Item]: return result - - Raises: - ApiNotEnabled: api is not invalid - """ - return self.adapter.search(*args, **kwargs) +# google-custom-seaerch - search + +from typing import List, AsyncGenerator + +from .types import Item +from .adapter import BaseAdapter + + +class CustomSearch: + """This is the class used when using Google Custom Search. + + Args: + adapter (BaseAdapter): Insert adapter + """ + APIURL: str = "https://www.googleapis.com/customsearch/v1" + + def __init__(self, adapter: BaseAdapter): + self.adapter = adapter + + def search(self, *args, **kwargs) -> List[Item]: + """This is searched using api. + + Args: + query (str): Search keyword + safe (bool): Using safe mode + filter_ (filter): Use filter mode + + Returns: + List[Item]: return result + + Raises: + ApiNotEnabled: api is not invalid + """ + return self.adapter.search(*args, **kwargs) + + async def asearch(self, *args, **kwargs) -> AsyncGenerator[Item, None]: + async for item in self.adapter.asearch(*args, **kwargs): + yield item