[feat] Add json log formatter with prometheus counter

fixcloudutils/logging/__init__.py

@@ -0,0 +1,62 @@
+#  Copyright (c) 2023. Some Engineering
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import logging
+import os
+from logging import StreamHandler, basicConfig
+from typing import Optional, List, Any
+
+from .json_logger import JsonFormatter
+from .prometheus_counter import PrometheusLoggingCounter
+
+__all__ = ["setup_logger", "JsonFormatter", "PrometheusLoggingCounter"]
+
+
+def setup_logger(
+    component: str,
+    *,
+    force: bool = True,
+    level: Optional[int] = None,
+    json_format: bool = True,
+    count_logs: bool = True,
+    log_format: Optional[str] = None,
+) -> List[StreamHandler[Any]]:
+    log_level = level or logging.INFO
+    # override log output via env var
+    plain_text = os.environ.get("LOG_TEXT", "false").lower() == "true"
+    handler = PrometheusLoggingCounter(component) if count_logs else StreamHandler()
+    if json_format and not plain_text:
+        formatter = JsonFormatter(
+            {
+                "timestamp": "asctime",
+                "level": "levelname",
+                "message": "message",
+                "pid": "process",
+                "thread": "threadName",
+            },
+            static_values={"component": component},
+        )
+        handler.setFormatter(formatter)
+        basicConfig(handlers=[handler], force=force, level=log_level)
+    else:
+        lf = log_format or "%(asctime)s %(levelname)s %(message)s"
+        basicConfig(handlers=[handler], format=lf, datefmt="%Y-%m-%dT%H:%M:%S", force=force, level=log_level)
+    return [handler]

fixcloudutils/logging/json_logger.py

@@ -0,0 +1,61 @@
+#  Copyright (c) 2023. Some Engineering
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import json
+from logging import Formatter, LogRecord
+from typing import Mapping, Optional, Dict
+
+
+class JsonFormatter(Formatter):
+    """
+    Simple json log formatter.
+    """
+
+    def __init__(
+        self,
+        fmt_dict: Mapping[str, str],
+        time_format: str = "%Y-%m-%dT%H:%M:%S",
+        static_values: Optional[Dict[str, str]] = None,
+    ) -> None:
+        super().__init__()
+        self.fmt_dict = fmt_dict
+        self.time_format = time_format
+        self.static_values = static_values or {}
+        self.__uses_time = "asctime" in self.fmt_dict.values()
+
+    def usesTime(self) -> bool:  # noqa: N802
+        return self.__uses_time
+
+    def format(self, record: LogRecord) -> str:
+        record.message = record.getMessage()
+        if self.__uses_time:
+            record.asctime = self.formatTime(record, self.time_format)
+
+        message_dict = {fmt_key: record.__dict__[fmt_val] for fmt_key, fmt_val in self.fmt_dict.items()}
+        message_dict.update(self.static_values)
+        if record.exc_info:
+            if not record.exc_text:
+                record.exc_text = self.formatException(record.exc_info)
+        if record.exc_text:
+            message_dict["exception"] = record.exc_text
+        if record.stack_info:
+            message_dict["stack_info"] = self.formatStack(record.stack_info)
+        return json.dumps(message_dict, default=str)

fixcloudutils/logging/prometheus_counter.py

@@ -0,0 +1,37 @@
+#  Copyright (c) 2023. Some Engineering
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from logging import StreamHandler, LogRecord
+from typing import Any
+
+from prometheus_client import Counter
+
+LogRecordCounter = Counter("log_record_counter", "Number of logs by severity", ["component", "level"])
+
+
+class PrometheusLoggingCounter(StreamHandler[Any]):
+    def __init__(self, component: str) -> None:
+        super().__init__()
+        self.component = component
+
+    def emit(self, record: LogRecord) -> None:
+        LogRecordCounter.labels(component=self.component, level=record.levelname).inc()
+        super().emit(record)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "fixcloudutils"
-version = "1.6.0"
+version = "1.7.0"
 authors = [{ name = "Some Engineering Inc." }]
 description = "Utilities for fixcloud."
 license = { file = "LICENSE" }

tests/logging_test.py

@@ -0,0 +1,43 @@
+#  Copyright (c) 2023. Some Engineering
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import logging
+
+import prometheus_client
+
+from fixcloudutils.logging import JsonFormatter, PrometheusLoggingCounter
+
+
+def test_json_logging() -> None:
+    format = JsonFormatter({"level": "levelname", "message": "message"})
+    record = logging.getLogger().makeRecord("test", logging.INFO, "test", 1, "test message", (), None)
+    assert format.format(record) == '{"level": "INFO", "message": "test message"}'
+
+
+def test_prometheus_counter() -> None:
+    counter = PrometheusLoggingCounter("test")
+    levels = {"CRITICAL": 50, "ERROR": 40, "WARNING": 30, "INFO": 20, "DEBUG": 10, "NOTSET": 0}
+    for level in levels.values():
+        record = logging.getLogger().makeRecord("test", level, "test", 1, "test message", (), None)
+        counter.emit(record)
+    gl = prometheus_client.generate_latest().decode("utf-8")
+    for level_name in levels.keys():
+        assert f'log_record_counter_total{{component="test",level="{level_name}"}}' in gl