Merge pull request #89 from AmiyaBot/dev

Dev

.github/workflows/pypi.yml

@@ -1,4 +1,4 @@
-name: Pylint
+name: Pypi
 
 on:
     push:

amiyabot/adapters/tencent/qqGroup/builder.py

@@ -123,6 +123,9 @@ async def get_image(self, image: Union[str, bytes]) -> Union[str, bytes]:
         return image
 
     async def get_voice(self, voice_file: str) -> str:
+        if voice_file.startswith('http'):
+            return voice_file
+
         voice = await silkcoder.async_encode(voice_file, ios_adaptive=True)
         path, url = self.temp_filename('silk')
 
@@ -132,6 +135,9 @@ async def get_voice(self, voice_file: str) -> str:
         return url
 
     async def get_video(self, video_file: str) -> str:
+        if video_file.startswith('http'):
+            return video_file
+
         path, url = self.temp_filename('mp4')
         shutil.copy(video_file, path)
         return url

amiyabot/builtin/lib/browserService/__init__.py

@@ -0,0 +1,83 @@
+from playwright.async_api import async_playwright, ConsoleMessage, Error as PageError
+
+from .launchConfig import *
+from .pagePool import *
+from .pageContext import PageContext
+
+
+class BrowserService:
+    def __init__(self):
+        self.playwright: Optional[Playwright] = None
+        self.browser: Optional[Union[Browser, BrowserContext]] = None
+        self.config: Optional[BrowserLaunchConfig] = None
+        self.pool: Optional[PagePool] = None
+
+        self.launched = False
+
+    def __str__(self):
+        return self.config.name if self.config else 'Not Launched'
+
+    async def launch(self, config: BrowserLaunchConfig):
+        if self.launched:
+            return None
+
+        self.launched = True
+
+        log.info('launching browser...')
+
+        self.playwright = await async_playwright().start()
+        self.browser = await config.launch_browser(self.playwright)
+        self.config = config
+
+        if self.config.page_pool_size:
+            self.pool = PagePool(self.browser, config)
+
+        # if not config.browser_name:
+        #     config.browser_name = self.browser._impl_obj._browser_type.name
+
+        log.info(f'{self} launched successful.')
+
+    async def close(self):
+        await self.browser.close()
+        await self.playwright.stop()
+
+        log.info(f'{self} closed.')
+
+    async def open_page(self, width: int, height: int):
+        if self.browser:
+            size = ViewportSize(width=width, height=height)
+
+            if self.pool:
+                page_context = await self.pool.acquire_page(size)
+            else:
+                page_context = PageContext(
+                    await self.browser.new_page(no_viewport=True, viewport=size),
+                )
+
+            if self.config.debug:
+                page_context.page.once('console', self.__console)
+                page_context.page.once('pageerror', self.__page_error)
+
+            hook_res = await self.config.on_page_created(page_context.page)
+            if hook_res:
+                page_context.page = hook_res
+
+            return page_context
+
+    @staticmethod
+    async def __console(message: ConsoleMessage):
+        text = f'console: {message.text}' + '\n    at {url}:{lineNumber}:{columnNumber}'.format(**message.location)
+
+        if message.type == 'warning':
+            log.warning(text)
+        elif message.type == 'error':
+            log.error(text)
+        else:
+            log.info(text)
+
+    @staticmethod
+    async def __page_error(error: PageError):
+        log.error(error.stack)
+
+
+basic_browser_service = BrowserService()

amiyabot/builtin/lib/browserService/launchConfig.py

@@ -0,0 +1,35 @@
+from typing import Union, Optional
+from playwright.async_api import Browser, BrowserType, BrowserContext, Page, Playwright
+from amiyabot.util import argv
+from amiyabot.log import LoggerManager
+
+DEFAULT_WIDTH = argv('browser-width', int) or 1280
+DEFAULT_HEIGHT = argv('browser-height', int) or 720
+DEFAULT_RENDER_TIME = argv('browser-render-time', int) or 200
+BROWSER_PAGE_POOL_SIZE = argv('browser-page-pool-size', int) or 0
+BROWSER_LAUNCH_WITH_HEADED = argv('browser-launch-with-headed', bool)
+
+log = LoggerManager('Browser')
+
+
+class BrowserLaunchConfig:
+    def __init__(self):
+        self.browser_type: str = 'chromium'
+        self.browser_name: [Optional] = None
+        self.page_pool_size: int = BROWSER_PAGE_POOL_SIZE
+        self.debug: bool = argv('debug', bool)
+
+    @property
+    def name(self):
+        return f'browser({self.browser_name or self.browser_type})'
+
+    async def launch_browser(self, playwright: Playwright) -> Union[Browser, BrowserContext]:
+        browser: BrowserType = getattr(playwright, self.browser_type)
+
+        return await browser.launch(headless=not BROWSER_LAUNCH_WITH_HEADED)
+
+    async def on_context_created(self, context: BrowserContext) -> Optional[BrowserContext]:
+        ...
+
+    async def on_page_created(self, page: Page) -> Optional[Page]:
+        ...

amiyabot/builtin/lib/browserService/pageContext.py

@@ -0,0 +1,12 @@
+from playwright.async_api import Page
+
+
+class PageContext:
+    def __init__(self, page: Page):
+        self.page = page
+
+    async def __aenter__(self):
+        return self.page
+
+    async def __aexit__(self, *args, **kwargs):
+        await self.page.close()

amiyabot/builtin/lib/browserService/pagePool.py

@@ -0,0 +1,98 @@
+import asyncio
+
+from playwright.async_api import ViewportSize
+from contextlib import asynccontextmanager
+
+from .launchConfig import *
+from .pageContext import PageContext
+
+
+class PagePool:
+    def __init__(self, browser: Union[Browser, BrowserContext], config: BrowserLaunchConfig):
+        self.config = config
+        self.browser = browser
+
+        self.size = 0
+        self.queuing_num = 0
+
+        self.lock = asyncio.Lock()
+        self.queue = asyncio.Queue()
+
+    @property
+    def max_size(self):
+        return self.config.page_pool_size
+
+    @property
+    def queue_size(self):
+        return self.queue.qsize()
+
+    @asynccontextmanager
+    async def __queuing(self):
+        self.queuing_num += 1
+        yield
+        self.queuing_num -= 1
+
+    async def acquire_page(self, viewport_size: ViewportSize):
+        log.debug(
+            f'{self.config.name} -- idle pages: {self.queue_size} opened: {self.size} queuing: {self.queuing_num}'
+        )
+
+        created = False
+
+        if self.queue_size == 0:
+            async with self.lock:
+                if self.size < self.max_size:
+                    if isinstance(self.browser, BrowserContext):
+                        page = await self.browser.new_page()
+                    else:
+                        context = await self.browser.new_context(no_viewport=True)
+                        hook_res = await self.config.on_context_created(context)
+                        if hook_res:
+                            context = hook_res
+
+                        page = await context.new_page()
+
+                    created = True
+                    self.size += 1
+
+                    log.debug(f'{self.config.name} -- page created. curr size: {self.size}/{self.max_size}')
+
+        if not created:
+            async with self.__queuing():
+                page: Page = await self.queue.get()
+
+        try:
+            await page.set_viewport_size(viewport_size)
+        except Exception as e:
+            log.warning(f'{self.config.name} -- {repr(e)}')
+            if 'context or browser has been closed' in str(e):
+                self.size -= 1
+                return await self.acquire_page(viewport_size)
+
+        return PagePoolContext(page, self)
+
+    async def release_page(self, page: Page):
+        try:
+            await page.context.clear_cookies()
+            await page.evaluate(
+                '''
+                localStorage.clear();
+                sessionStorage.clear();
+                '''
+            )
+            await page.goto('about:blank')
+            await self.queue.put(page)
+        except Exception as e:
+            log.warning(f'{self.config.name} -- {repr(e)}')
+            self.size -= 1
+
+        log.debug(f'{self.config.name} -- page released. idle pages: {self.queue_size}')
+
+
+class PagePoolContext(PageContext):
+    def __init__(self, page: Page, pool: PagePool):
+        super().__init__(page)
+        self.pool = pool
+
+    async def __aexit__(self, *args, **kwargs):
+        await self.pool.release_page(self.page)