From 3412c9cb217dda3ef3c309c3bc7e03a74737cf10 Mon Sep 17 00:00:00 2001 From: Matthias Veit Date: Mon, 2 Oct 2023 19:56:53 +0200 Subject: [PATCH] [feat] Add json log formatter with prometheus counter --- fixcloudutils/logging/__init__.py | 62 +++++++++++++++++++++ fixcloudutils/logging/json_logger.py | 61 ++++++++++++++++++++ fixcloudutils/logging/prometheus_counter.py | 37 ++++++++++++ pyproject.toml | 2 +- tests/logging_test.py | 43 ++++++++++++++ 5 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 fixcloudutils/logging/__init__.py create mode 100644 fixcloudutils/logging/json_logger.py create mode 100644 fixcloudutils/logging/prometheus_counter.py create mode 100644 tests/logging_test.py diff --git a/fixcloudutils/logging/__init__.py b/fixcloudutils/logging/__init__.py new file mode 100644 index 0000000..122fd80 --- /dev/null +++ b/fixcloudutils/logging/__init__.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023. Some Engineering +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import logging +import os +from logging import StreamHandler, basicConfig +from typing import Optional, List, Any + +from .json_logger import JsonFormatter +from .prometheus_counter import PrometheusLoggingCounter + +__all__ = ["setup_logger", "JsonFormatter", "PrometheusLoggingCounter"] + + +def setup_logger( + component: str, + *, + force: bool = True, + level: Optional[int] = None, + json_format: bool = True, + count_logs: bool = True, + log_format: Optional[str] = None, +) -> List[StreamHandler[Any]]: + log_level = level or logging.INFO + # override log output via env var + plain_text = os.environ.get("LOG_TEXT", "false").lower() == "true" + handler = PrometheusLoggingCounter(component) if count_logs else StreamHandler() + if json_format and not plain_text: + formatter = JsonFormatter( + { + "timestamp": "asctime", + "level": "levelname", + "message": "message", + "pid": "process", + "thread": "threadName", + }, + static_values={"component": component}, + ) + handler.setFormatter(formatter) + basicConfig(handlers=[handler], force=force, level=log_level) + else: + lf = log_format or "%(asctime)s %(levelname)s %(message)s" + basicConfig(handlers=[handler], format=lf, datefmt="%Y-%m-%dT%H:%M:%S", force=force, level=log_level) + return [handler] diff --git a/fixcloudutils/logging/json_logger.py b/fixcloudutils/logging/json_logger.py new file mode 100644 index 0000000..6c8b120 --- /dev/null +++ b/fixcloudutils/logging/json_logger.py @@ -0,0 +1,61 @@ +# Copyright (c) 2023. Some Engineering +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import json +from logging import Formatter, LogRecord +from typing import Mapping, Optional, Dict + + +class JsonFormatter(Formatter): + """ + Simple json log formatter. + """ + + def __init__( + self, + fmt_dict: Mapping[str, str], + time_format: str = "%Y-%m-%dT%H:%M:%S", + static_values: Optional[Dict[str, str]] = None, + ) -> None: + super().__init__() + self.fmt_dict = fmt_dict + self.time_format = time_format + self.static_values = static_values or {} + self.__uses_time = "asctime" in self.fmt_dict.values() + + def usesTime(self) -> bool: # noqa: N802 + return self.__uses_time + + def format(self, record: LogRecord) -> str: + record.message = record.getMessage() + if self.__uses_time: + record.asctime = self.formatTime(record, self.time_format) + + message_dict = {fmt_key: record.__dict__[fmt_val] for fmt_key, fmt_val in self.fmt_dict.items()} + message_dict.update(self.static_values) + if record.exc_info: + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + if record.exc_text: + message_dict["exception"] = record.exc_text + if record.stack_info: + message_dict["stack_info"] = self.formatStack(record.stack_info) + return json.dumps(message_dict, default=str) diff --git a/fixcloudutils/logging/prometheus_counter.py b/fixcloudutils/logging/prometheus_counter.py new file mode 100644 index 0000000..0d406fa --- /dev/null +++ b/fixcloudutils/logging/prometheus_counter.py @@ -0,0 +1,37 @@ +# Copyright (c) 2023. Some Engineering +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from logging import StreamHandler, LogRecord +from typing import Any + +from prometheus_client import Counter + +LogRecordCounter = Counter("log_record_counter", "Number of logs by severity", ["component", "level"]) + + +class PrometheusLoggingCounter(StreamHandler[Any]): + def __init__(self, component: str) -> None: + super().__init__() + self.component = component + + def emit(self, record: LogRecord) -> None: + LogRecordCounter.labels(component=self.component, level=record.levelname).inc() + super().emit(record) diff --git a/pyproject.toml b/pyproject.toml index a7d8183..af2f07a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "fixcloudutils" -version = "1.6.0" +version = "1.7.0" authors = [{ name = "Some Engineering Inc." }] description = "Utilities for fixcloud." license = { file = "LICENSE" } diff --git a/tests/logging_test.py b/tests/logging_test.py new file mode 100644 index 0000000..d92dfda --- /dev/null +++ b/tests/logging_test.py @@ -0,0 +1,43 @@ +# Copyright (c) 2023. Some Engineering +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import logging + +import prometheus_client + +from fixcloudutils.logging import JsonFormatter, PrometheusLoggingCounter + + +def test_json_logging() -> None: + format = JsonFormatter({"level": "levelname", "message": "message"}) + record = logging.getLogger().makeRecord("test", logging.INFO, "test", 1, "test message", (), None) + assert format.format(record) == '{"level": "INFO", "message": "test message"}' + + +def test_prometheus_counter() -> None: + counter = PrometheusLoggingCounter("test") + levels = {"CRITICAL": 50, "ERROR": 40, "WARNING": 30, "INFO": 20, "DEBUG": 10, "NOTSET": 0} + for level in levels.values(): + record = logging.getLogger().makeRecord("test", level, "test", 1, "test message", (), None) + counter.emit(record) + gl = prometheus_client.generate_latest().decode("utf-8") + for level_name in levels.keys(): + assert f'log_record_counter_total{{component="test",level="{level_name}"}}' in gl