Switch to using JSON for logging

CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 2024-10-02
+
+### Changed
+
+* Use [python-json-logger](https://github.com/madzak/python-json-logger) and [readable-log-formatter](https://github.com/ipmb/readable-log-formatter) for for better log parsing with JSON
+
 ## 2024-03-16
 
 ### Added

README.md

@@ -38,6 +38,7 @@ features.
   Docker image
 * [UV](https://github.com/astral-sh/uv) - Used to maintain python requirements
 * [Just](https://github.com/casey/just) - Popular tool for running common commands (make equivalent)
+* [python-json-logger](https://github.com/madzak/python-json-logger) and [readable-log-formatter](https://github.com/ipmb/readable-log-formatter) - Use JSON logging for better log parsing
 
 ### 📦️ Django Packages
 

config/requirements/dev.in

@@ -15,5 +15,6 @@ mkdocs-material
 mkdocstrings[python]
 model-bakery
 mypy
+readable-log-formatter
 ruff
 ruff-lsp

config/requirements/dev_lock.txt

@@ -1130,6 +1130,10 @@ python-fsutil==0.14.1 \
     --hash=sha256:0d45e623f0f4403f674bdd8ae7aa7d24a4b3132ea45c65416bd2865e6b20b035 \
     --hash=sha256:8fb204fa8059f37bdeee8a1dc0fff010170202ea47c4225ee71bb3c26f3997be
     # via django-maintenance-mode
+python-json-logger==2.0.7 \
+    --hash=sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c \
+    --hash=sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd
+    # via -r config/requirements/prod.in
 pyyaml==6.0.2 \
     --hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \
     --hash=sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48 \
@@ -1194,6 +1198,10 @@ pyyaml-env-tag==0.1 \
     --hash=sha256:70092675bda14fdec33b31ba77e7543de9ddc88f2e5b99160396572d11525bdb \
     --hash=sha256:af31106dec8a4d68c60207c1886031cbf839b68aa7abccdb19868200532c2069
     # via mkdocs
+readable-log-formatter==0.1.4 \
+    --hash=sha256:3cbbd4737074e1ffa13d62c68fa5d10019737971ac33656fe328766e18e72a89 \
+    --hash=sha256:b9054caa2b3ad4d3e49f7d0dda5f53955396912d46a7b14b2eda814e4a1f4550
+    # via -r config/requirements/dev.in
 redis==5.1.0 \
     --hash=sha256:b756df1e4a3858fcc0ef861f3fc53623a96c41e2b1f5304e09e0fe758d333d40 \
     --hash=sha256:fd4fccba0d7f6aa48c58a78d76ddb4afc698f5da4a2c1d03d916e4fd7ab88cdd

config/requirements/prod.in

@@ -16,5 +16,6 @@ hiredis~=3.0  # needed for Django's native cache and sessions
 psycopg2-binary~=2.8
 pytest-cov~=5.0
 pytest-django~=4.5
+python-json-logger~=2.0
 redis~=5.0  # needed for celery
 uwsgi~=2.0

config/requirements/prod_lock.txt

@@ -402,6 +402,10 @@ python-fsutil==0.14.1 \
     --hash=sha256:0d45e623f0f4403f674bdd8ae7aa7d24a4b3132ea45c65416bd2865e6b20b035 \
     --hash=sha256:8fb204fa8059f37bdeee8a1dc0fff010170202ea47c4225ee71bb3c26f3997be
     # via django-maintenance-mode
+python-json-logger==2.0.7 \
+    --hash=sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c \
+    --hash=sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd
+    # via -r config/requirements/prod.in
 redis==5.1.0 \
     --hash=sha256:b756df1e4a3858fcc0ef861f3fc53623a96c41e2b1f5304e09e0fe758d333d40 \
     --hash=sha256:fd4fccba0d7f6aa48c58a78d76ddb4afc698f5da4a2c1d03d916e4fd7ab88cdd

config/settings/_base.py

@@ -1,3 +1,4 @@
+import contextlib
 import socket
 
 import environs
@@ -245,56 +246,62 @@
 EMAIL_HOST_USER = email["EMAIL_HOST_USER"]
 EMAIL_USE_TLS = email["EMAIL_USE_TLS"]
 
-DEBUG_LOGGING = {
-    "version": 1,
-    "disable_existing_loggers": False,
-    "formatters": {
-        "verbose": {
-            "format": "{levelname} - {asctime} - {module} - {message}",
-            "style": "{",
-        },
-    },
-    "handlers": {
-        "console": {
-            "class": "logging.StreamHandler",
-            "formatter": "verbose",
-        },
-    },
-    "loggers": {
-        "django.request": {
-            "level": "DEBUG",
-            "handlers": ["console"],
-            "propagate": False,
-        },
-    },
-}
 
-PROD_LOGGING = {
+def log_format() -> str:
+    """Dump all available values into the JSON log output."""
+    keys = (
+        "asctime",
+        "created",
+        "levelname",
+        "levelno",
+        "filename",
+        "funcName",
+        "lineno",
+        "module",
+        "message",
+        "name",
+        "pathname",
+        "process",
+        "processName",
+    )
+    return " ".join([f"%({i:s})" for i in keys])
+
+
+log_level = "WARNING"
+IS_DEBUG_LOGGING_ON = env.bool("IS_DEBUG_LOGGING_ON", default=False)
+if IS_DEBUG_LOGGING_ON is True:
+    log_level = "DEBUG"
+
+LOGGING = {
     "version": 1,
     "disable_existing_loggers": False,
     "formatters": {
-        "verbose": {"format": "%(levelname)s - %(asctime)s - %(module)s - %(message)s"},
+        "default": {
+            "format": log_format(),
+            "class": "pythonjsonlogger.jsonlogger.JsonFormatter",
+        },
     },
     "handlers": {
+        # console logs to stderr
         "console": {
             "class": "logging.StreamHandler",
-            "formatter": "verbose",
+            "formatter": "default",
         },
     },
     "loggers": {
-        "django.request": {
-            "level": "ERROR",
+        # default for all Python modules not listed below
+        "": {
+            "level": log_level,
             "handlers": ["console"],
-            "propagate": False,
         },
     },
 }
 
-IS_DEBUG_LOGGING_ON = env.bool("IS_DEBUG_LOGGING_ON", default=False)
-LOGGING = PROD_LOGGING
+# setup pretty logging for local dev
+with contextlib.suppress(ModuleNotFoundError):
+    import readable_log_formatter  # noqa: F401
 
-if IS_DEBUG_LOGGING_ON is True:
-    LOGGING = DEBUG_LOGGING
+    LOGGING["formatters"]["default"]["class"] = "readable_log_formatter.ReadableFormatter"  # type: ignore
 
 # MAINTENANCE MODE SETTINGS
 MAINTENANCE_MODE_STATE_BACKEND = "maintenance_mode.backends.CacheBackend"