From 19451821842629c8f5fb45bcac4c690df4a7230d Mon Sep 17 00:00:00 2001 From: Paolo Quadri Date: Wed, 27 Mar 2024 11:00:04 +0100 Subject: [PATCH] feat: first implementation --- .github/workflows/release.yml | 24 ++++++ .github/workflows/tests.yml | 55 +++++++++++++ README.md | 11 +++ poetry.lock | 127 +++++++++++++++++++++++++++++++ pyproject.toml | 17 +++++ remote_log_formatter/__init__.py | 107 ++++++++++++++++++++++++++ remote_log_formatter/__main__.py | 5 ++ tests/__init__.py | 0 tests/test_formatter.py | 97 +++++++++++++++++++++++ 9 files changed, 443 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/tests.yml create mode 100644 README.md create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 remote_log_formatter/__init__.py create mode 100644 remote_log_formatter/__main__.py create mode 100644 tests/__init__.py create mode 100644 tests/test_formatter.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a40daf5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,24 @@ +name: Semantic Release + +on: + push: + branches: + - main + +jobs: + release: + runs-on: ubuntu-latest + concurrency: release + permissions: + id-token: write + contents: write + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Python Semantic Release + uses: python-semantic-release/python-semantic-release@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..61adf8c --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,55 @@ +on: + push: + branches: + - "**" + tags-ignore: + - "*.*.*" + +name: tests + +concurrency: + group: tests + +jobs: + tests: + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + poetry-version: ["1.8.2"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + name: Checkout + + - uses: actions/setup-python@v4 + name: Setup Python + with: + python-version: ${{ matrix.python-version }} + + - uses: abatilo/actions-poetry@v3 + name: Install Poetry + with: + poetry-version: ${{ matrix.poetry-version }} + + - name: Update Poetry cache location + run: poetry config virtualenvs.in-project true + + - id: venv_cache + uses: actions/cache@v3 + name: Cache or Restore venv + with: + path: .venv + key: venv-${{ matrix.python-version }}-${{ matrix.poetry-version }}-lock-${{ hashFiles('poetry.lock') }} + + - name: Install Poetry Dependencies + run: poetry install + if: steps.venv_cache.outputs.cache-hit != 'true' + + - name: Run ruff + run: poetry run ruff check . + + - name: Run ruff format + run: poetry run ruff format --check . + + - name: Run tests + run: poetry run pytest diff --git a/README.md b/README.md new file mode 100644 index 0000000..b8d9c64 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# Remote Python log formatter + +## Sample usage + +```python +from remote_log_formatter import get_logger, setup_logging + +setup_logging(json=False) +logger = get_logger(__package__) +logger.info("foo") +``` diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..22183df --- /dev/null +++ b/poetry.lock @@ -0,0 +1,127 @@ +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "packaging" +version = "24.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, + {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, +] + +[[package]] +name = "pluggy" +version = "1.4.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, + {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pytest" +version = "8.1.1" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"}, + {file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.4,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "ruff" +version = "0.3.4" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.3.4-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:60c870a7d46efcbc8385d27ec07fe534ac32f3b251e4fc44b3cbfd9e09609ef4"}, + {file = "ruff-0.3.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6fc14fa742e1d8f24910e1fff0bd5e26d395b0e0e04cc1b15c7c5e5fe5b4af91"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3ee7880f653cc03749a3bfea720cf2a192e4f884925b0cf7eecce82f0ce5854"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf133dd744f2470b347f602452a88e70dadfbe0fcfb5fd46e093d55da65f82f7"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f3860057590e810c7ffea75669bdc6927bfd91e29b4baa9258fd48b540a4365"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:986f2377f7cf12efac1f515fc1a5b753c000ed1e0a6de96747cdf2da20a1b369"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fd98e85869603e65f554fdc5cddf0712e352fe6e61d29d5a6fe087ec82b76c"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64abeed785dad51801b423fa51840b1764b35d6c461ea8caef9cf9e5e5ab34d9"}, + {file = "ruff-0.3.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df52972138318bc7546d92348a1ee58449bc3f9eaf0db278906eb511889c4b50"}, + {file = "ruff-0.3.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:98e98300056445ba2cc27d0b325fd044dc17fcc38e4e4d2c7711585bd0a958ed"}, + {file = "ruff-0.3.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:519cf6a0ebed244dce1dc8aecd3dc99add7a2ee15bb68cf19588bb5bf58e0488"}, + {file = "ruff-0.3.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:bb0acfb921030d00070539c038cd24bb1df73a2981e9f55942514af8b17be94e"}, + {file = "ruff-0.3.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cf187a7e7098233d0d0c71175375c5162f880126c4c716fa28a8ac418dcf3378"}, + {file = "ruff-0.3.4-py3-none-win32.whl", hash = "sha256:af27ac187c0a331e8ef91d84bf1c3c6a5dea97e912a7560ac0cef25c526a4102"}, + {file = "ruff-0.3.4-py3-none-win_amd64.whl", hash = "sha256:de0d5069b165e5a32b3c6ffbb81c350b1e3d3483347196ffdf86dc0ef9e37dd6"}, + {file = "ruff-0.3.4-py3-none-win_arm64.whl", hash = "sha256:6810563cc08ad0096b57c717bd78aeac888a1bfd38654d9113cb3dc4d3f74232"}, + {file = "ruff-0.3.4.tar.gz", hash = "sha256:f0f4484c6541a99862b693e13a151435a279b271cff20e37101116a21e2a1ad1"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "df30f5829e1baef80fe3d96327705f833f787a2bb43a60ed01b64559326e8b57" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..04f5319 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "remote-log-formatter" +version = "0.1.0" +description = "" +authors = ["Paolo Quadri "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" + +[tool.poetry.group.dev.dependencies] +ruff = "^0.3.4" +pytest = "^8.1.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/remote_log_formatter/__init__.py b/remote_log_formatter/__init__.py new file mode 100644 index 0000000..608a0ec --- /dev/null +++ b/remote_log_formatter/__init__.py @@ -0,0 +1,107 @@ +import json +import logging +import logging.config +import os +import sys +from datetime import datetime + + +class SimpleFormatter(logging.Formatter): + def __init__( + self, fmt=None, datefmt=None, style="%", validate=True, *, defaults=None + ) -> None: + if fmt is None: + fmt = "%(asctime)s | %(levelname)s | %(message)s | %(pathname)s:%(lineno)s" + super(SimpleFormatter, self).__init__(fmt, datefmt, style, validate) + + +class JSONFormatter(logging.Formatter): + def __init__(self, *args, **kwargs) -> None: + super(JSONFormatter, self).__init__(*args, **kwargs) + + self._pid = os.getpid() + + @staticmethod + def format_timestamp(time: float) -> str: + return datetime.fromtimestamp(time).isoformat() + + def format(self, record: logging.LogRecord) -> str: + try: + msg = record.msg % record.args + except TypeError: + msg = record.msg + + extra = {"type": "log"} + + exc = "" + if record.exc_info: + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + extra["class"] = str(record.exc_info[0]) + + if record.exc_text: + if exc[-1:] != "\n": + exc += "\n" + exc += record.exc_text + + if record.stack_info: + if exc[-1:] != "\n": + exc += "\n" + exc += self.formatStack(record.stack_info) + + if len(exc): + extra["traceback"] = exc + extra["type"] = "exception" + + message = { + "datetime": self.format_timestamp(record.created), + "level": record.levelname, + "message": msg, + "channel": record.name, + "pid": record.process, + "context": { + "processname": record.processName, + "pathname": record.pathname, + "module": record.module, + "function": record.funcName, + "lineno": record.lineno, + }, + "extra": extra, + } + + return json.dumps(message, default=str) + + +def setup_logging(json: bool = True) -> None: + LOG_CONFIG = dict( + version=1, + disable_existing_loggers=False, + root={ + "level": "INFO", + "handlers": ["console"], + }, + loggers={ + "root": { + "level": "INFO", + "handlers": ["console"], + "propagate": True, + } + }, + handlers={ + "console": { + "class": "logging.StreamHandler", + "formatter": "generic" if json else "simple", + "stream": sys.stdout, + }, + }, + formatters={ + "generic": {"class": "remote_log_formatter.JSONFormatter"}, + "simple": {"class": "remote_log_formatter.SimpleFormatter"}, + }, + ) + + logging.config.dictConfig(LOG_CONFIG) + + +def get_logger(name: str = "remote") -> logging.Logger: + return logging.getLogger(name) diff --git a/remote_log_formatter/__main__.py b/remote_log_formatter/__main__.py new file mode 100644 index 0000000..1e4acea --- /dev/null +++ b/remote_log_formatter/__main__.py @@ -0,0 +1,5 @@ +from remote_log_formatter import get_logger, setup_logging + +setup_logging(json=False) +logger = get_logger(__package__) +logger.info("foo") diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_formatter.py b/tests/test_formatter.py new file mode 100644 index 0000000..9c27ddb --- /dev/null +++ b/tests/test_formatter.py @@ -0,0 +1,97 @@ +import json +import logging +from dataclasses import dataclass +from decimal import Decimal +from functools import lru_cache +from typing import Any + +import pytest + +from remote_log_formatter import JSONFormatter, SimpleFormatter +from remote_log_formatter import get_logger as _logger +from remote_log_formatter import setup_logging as _setup + + +@lru_cache() +@pytest.fixture() +def logger_json() -> None: + _setup() + return _logger() + + +@lru_cache() +@pytest.fixture() +def logger_plain() -> None: + _setup(json=False) + return _logger() + + +@dataclass(frozen=True) +class Foo: + bar: str + + +@lru_cache() +@pytest.mark.parametrize( + "message,expected", + [ + pytest.param("foo", "foo", id="str"), + pytest.param(42, 42, id="integer"), + pytest.param(Decimal(42), "42", id="decimal"), + pytest.param(42.0, 42.0, id="float"), + pytest.param(Foo(bar=42), "Foo(bar=42)", id="dataclass"), + ], +) +def test_json_logger( + logger_json: logging.Logger, + caplog: pytest.LogCaptureFixture, + message: Any, + expected: str, +) -> None: + logger_json.info(message) + assert caplog.records + r = caplog.records[0] + + data = json.loads(JSONFormatter().format(r)) + assert data == { + "datetime": data["datetime"], + "level": "INFO", + "message": expected, + "channel": "remote", + "pid": data["pid"], + "context": { + "processname": "MainProcess", + "pathname": data["context"]["pathname"], + "module": "test_formatter", + "function": "test_json_logger", + "lineno": data["context"]["lineno"], + }, + "extra": {"type": "log"}, + } + + +@lru_cache() +@pytest.mark.parametrize( + "message,expected", + [ + pytest.param("foo", "foo", id="str"), + pytest.param(42, "42", id="integer"), + pytest.param(Decimal(42), "42", id="decimal"), + pytest.param(42.0, "42.0", id="float"), + pytest.param(Foo(bar=42), "Foo(bar=42)", id="dataclass"), + ], +) +def test_plain_logger( + logger_plain: logging.Logger, + caplog: pytest.LogCaptureFixture, + message: Any, + expected: str, +) -> None: + logger_plain.info(message) + assert caplog.records + r = caplog.records[0] + data = SimpleFormatter().format(r) + + _ts, level, message, path = data.split("|") + assert message.strip() == expected + assert level.strip() == "INFO"