From f1b2c716c3a07046520bc3f19741021f9014e63e Mon Sep 17 00:00:00 2001 From: Adam Hopkins Date: Wed, 28 Sep 2022 09:18:39 +0300 Subject: [PATCH] Add logging process extension (#134) --- sanic_ext/bootstrap.py | 10 ++- sanic_ext/config.py | 4 + sanic_ext/extensions/base.py | 5 ++ sanic_ext/extensions/health/extension.py | 3 + sanic_ext/extensions/logging/__init__.py | 0 sanic_ext/extensions/logging/extension.py | 24 ++++++ sanic_ext/extensions/logging/logger.py | 96 +++++++++++++++++++++++ sanic_ext/extensions/openapi/extension.py | 3 + 8 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 sanic_ext/extensions/logging/__init__.py create mode 100644 sanic_ext/extensions/logging/extension.py create mode 100644 sanic_ext/extensions/logging/logger.py diff --git a/sanic_ext/bootstrap.py b/sanic_ext/bootstrap.py index c0aae1e..6193b5c 100644 --- a/sanic_ext/bootstrap.py +++ b/sanic_ext/bootstrap.py @@ -15,6 +15,7 @@ from sanic_ext.extensions.http.extension import HTTPExtension from sanic_ext.extensions.injection.extension import InjectionExtension from sanic_ext.extensions.injection.registry import InjectionRegistry +from sanic_ext.extensions.logging.extension import LoggingExtension from sanic_ext.extensions.openapi.builders import SpecificationBuilder from sanic_ext.extensions.openapi.extension import OpenAPIExtension from sanic_ext.utils.string import camel_to_snake @@ -91,6 +92,7 @@ def __init__( OpenAPIExtension, HTTPExtension, HealthExtension, + LoggingExtension, ] ) @@ -98,7 +100,7 @@ def __init__( extensions.append(TemplatingExtension) started = set() - for ext in extensions[::-1]: + for ext in extensions: if ext in started: continue extension = Extension.create(ext, app, self.config) @@ -109,9 +111,9 @@ def __init__( def _display(self): init_logs = ["Sanic Extensions:"] for extension in self.extensions: - init_logs.append( - f" > {extension.name} {extension.render_label()}" - ) + label = extension.render_label() + if extension.included(): + init_logs.append(f" > {extension.name} {label}") list(map(logger.info, init_logs)) diff --git a/sanic_ext/config.py b/sanic_ext/config.py index 8dece6e..93775d2 100644 --- a/sanic_ext/config.py +++ b/sanic_ext/config.py @@ -36,6 +36,8 @@ def __init__( http_auto_options: bool = True, http_auto_trace: bool = False, injection_signal: Union[str, Event] = Event.HTTP_ROUTING_AFTER, + logging: bool = False, + logging_queue_max_size: int = 4096, oas: bool = True, oas_autodoc: bool = True, oas_ignore_head: bool = True, @@ -83,6 +85,8 @@ def __init__( self.HTTP_AUTO_OPTIONS = http_auto_options self.HTTP_AUTO_TRACE = http_auto_trace self.INJECTION_SIGNAL = injection_signal + self.LOGGING = logging + self.LOGGING_QUEUE_MAX_SIZE = logging_queue_max_size self.OAS = oas self.OAS_AUTODOC = oas_autodoc self.OAS_IGNORE_HEAD = oas_ignore_head diff --git a/sanic_ext/extensions/base.py b/sanic_ext/extensions/base.py index fa03faf..ad5fa67 100644 --- a/sanic_ext/extensions/base.py +++ b/sanic_ext/extensions/base.py @@ -53,11 +53,16 @@ def label(self): return "" def render_label(self): + if not self.included: + return "~~disabled~~" label = self.label() if not label: return "" return f"[{label}]" + def included(self): + return True + @classmethod def create( cls, diff --git a/sanic_ext/extensions/health/extension.py b/sanic_ext/extensions/health/extension.py index 9f31f23..e9e742a 100644 --- a/sanic_ext/extensions/health/extension.py +++ b/sanic_ext/extensions/health/extension.py @@ -25,3 +25,6 @@ def startup(self, bootstrap) -> None: if self.config.HEALTH_ENDPOINT: setup_health_endpoint(self.app) + + def included(self): + return self.config.HEALTH diff --git a/sanic_ext/extensions/logging/__init__.py b/sanic_ext/extensions/logging/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sanic_ext/extensions/logging/extension.py b/sanic_ext/extensions/logging/extension.py new file mode 100644 index 0000000..d31f79d --- /dev/null +++ b/sanic_ext/extensions/logging/extension.py @@ -0,0 +1,24 @@ +from sanic.exceptions import SanicException + +from ..base import Extension +from .logger import Logger + + +class LoggingExtension(Extension): + name = "logging" + MIN_VERSION = (22, 9) + + def startup(self, bootstrap) -> None: + if self.included(): + if self.MIN_VERSION > bootstrap.sanic_version: + min_version = ".".join(map(str, self.MIN_VERSION)) + sanic_version = ".".join(map(str, bootstrap.sanic_version)) + raise SanicException( + f"The logging extension only works with Sanic " + f"v{min_version} and above. It looks like you are " + f"running {sanic_version}." + ) + Logger.setup(self.app) + + def included(self): + return self.config.LOGGING diff --git a/sanic_ext/extensions/logging/logger.py b/sanic_ext/extensions/logging/logger.py new file mode 100644 index 0000000..2cc4f97 --- /dev/null +++ b/sanic_ext/extensions/logging/logger.py @@ -0,0 +1,96 @@ +from collections import defaultdict +from logging import LogRecord +from logging.handlers import QueueHandler +from multiprocessing import Manager +from queue import Empty, Full +from signal import SIGINT, SIGTERM +from signal import signal as signal_func + +from sanic import Sanic +from sanic.log import access_logger, error_logger +from sanic.log import logger as root_logger +from sanic.log import server_logger + + +async def prepare_logger(app: Sanic, *_): + Logger.prepare(app) + + +async def setup_logger(app: Sanic, *_): + logger = Logger() + app.manager.manage( + "Logger", + logger, + { + "queue": app.shared_ctx.logger_queue, + }, + ) + + +class SanicQueueHandler(QueueHandler): + def emit(self, record: LogRecord) -> None: + try: + return super().enqueue(record) + except Full: + server_logger.warning( + "Background logger is full. Emitting log in process." + ) + server_logger.handle(record) + + +async def setup_server_logging(app: Sanic): + qhandler = SanicQueueHandler(app.shared_ctx.logger_queue) + app.ctx._logger_handlers = defaultdict(list) + app.ctx._qhandler = qhandler + + for logger_instance in (root_logger, access_logger, error_logger): + for handler in logger_instance.handlers: + logger_instance.removeHandler(handler) + logger_instance.addHandler(qhandler) + + +async def remove_server_logging(app: Sanic): + for logger, handlers in app.ctx._logger_handlers.items(): + logger.removeHandler(app.ctx._qhandler) + for handler in handlers: + logger.addHandler(handler) + + +class Logger: + def __init__(self): + self.run = True + self.loggers = { + logger.name: logger + for logger in (root_logger, access_logger, error_logger) + } + + def __call__(self, queue) -> None: + signal_func(SIGINT, self.stop) + signal_func(SIGTERM, self.stop) + + while self.run: + try: + record: LogRecord = queue.get_nowait() + except Empty: + continue + logger = self.loggers.get(record.name) + logger.handle(record) + + def stop(self, *_): + if self.run: + self.run = False + + @classmethod + def prepare(cls, app: Sanic): + sync_manager = Manager() + logger_queue = sync_manager.Queue( + maxsize=app.config.LOGGING_QUEUE_MAX_SIZE + ) + app.shared_ctx.logger_queue = logger_queue + + @classmethod + def setup(cls, app: Sanic): + app.main_process_start(prepare_logger) + app.main_process_ready(setup_logger) + app.before_server_start(setup_server_logging) + app.before_server_stop(remove_server_logging) diff --git a/sanic_ext/extensions/openapi/extension.py b/sanic_ext/extensions/openapi/extension.py index 05e3b74..c2aef9e 100644 --- a/sanic_ext/extensions/openapi/extension.py +++ b/sanic_ext/extensions/openapi/extension.py @@ -26,6 +26,9 @@ def label(self): return "" + def included(self): + return self.config.OAS + def _make_url(self): name = f"{self.bp.name}.index" _server = (