From 7df78d6de0fbedab30df9fb5c65636798988b4a3 Mon Sep 17 00:00:00 2001 From: MEHRSHAD MIRSHEKARY Date: Fri, 4 Oct 2024 13:34:09 +0330 Subject: [PATCH] :zap::hammer: refactor(constants): Add new config_types and context placeholder - Added new configs in default settings - Added new type alias for configs in config types - Added context placeholder in format specifiers - Updated conf to handle new formatters & configs --- django_logging/constants/__init__.py | 4 ++ django_logging/constants/config_types.py | 32 ++++++--- django_logging/constants/default_settings.py | 42 ++++++++++-- .../constants/log_format_specifiers.py | 1 + django_logging/settings/conf.py | 68 +++++++++++++++---- django_logging/utils/get_conf.py | 44 +++++++++++- django_logging/utils/set_conf.py | 28 +++++--- .../validators/config_validators.py | 8 +-- .../validators/email_settings_validator.py | 2 +- 9 files changed, 185 insertions(+), 44 deletions(-) diff --git a/django_logging/constants/__init__.py b/django_logging/constants/__init__.py index 260cfc7..b8af46f 100644 --- a/django_logging/constants/__init__.py +++ b/django_logging/constants/__init__.py @@ -2,3 +2,7 @@ from .default_settings import DefaultConsoleSettings, DefaultLoggingSettings from .log_format_options import FORMAT_OPTIONS from .log_format_specifiers import LOG_FORMAT_SPECIFIERS + +# Used in settings.conf +ALLOWED_EXTRA_FILE_TYPES = ["JSON", "XML"] +ALLOWED_FILE_FORMAT_TYPES = ["JSON", "XML", "FLAT"] diff --git a/django_logging/constants/config_types.py b/django_logging/constants/config_types.py index f5b135c..9571764 100644 --- a/django_logging/constants/config_types.py +++ b/django_logging/constants/config_types.py @@ -2,8 +2,16 @@ FormatOption = Union[int, str] +# Type Aliases for configurations +LogFileFormatType = Literal["JSON", "XML", "FLAT", "LOG"] +LogLevel = Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] +LogDir = str +LogLevels = List[LogLevel] +NotifierLogLevels = List[Literal["ERROR", "CRITICAL"]] +LogDateFormat = str + -class LogEmailNotifierType(TypedDict, total=False): +class LogEmailNotifier(TypedDict, total=False): ENABLE: bool NOTIFY_ERROR: bool NOTIFY_CRITICAL: bool @@ -11,7 +19,7 @@ class LogEmailNotifierType(TypedDict, total=False): USE_TEMPLATE: bool -class LogFileFormatsType(TypedDict, total=False): +class LogFileFormats(TypedDict, total=False): DEBUG: FormatOption INFO: FormatOption WARNING: FormatOption @@ -19,9 +27,17 @@ class LogFileFormatsType(TypedDict, total=False): CRITICAL: FormatOption -# Type Aliases for other configurations -LogLevel = Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] -LogDir = str -LogLevels = List[LogLevel] -NotifierLogLevels = List[Literal["ERROR", "CRITICAL"]] -LogDateFormat = str +class LogFileFormatTypes(TypedDict, total=False): + DEBUG: LogFileFormatType + INFO: LogFileFormatType + WARNING: LogFileFormatType + ERROR: LogFileFormatType + CRITICAL: LogFileFormatType + + +class ExtraLogFiles(TypedDict, total=False): + DEBUG: bool + INFO: bool + WARNING: bool + ERROR: bool + CRITICAL: bool diff --git a/django_logging/constants/default_settings.py b/django_logging/constants/default_settings.py index 06e3f2d..b38a036 100644 --- a/django_logging/constants/default_settings.py +++ b/django_logging/constants/default_settings.py @@ -3,28 +3,33 @@ from typing import cast from django_logging.constants.config_types import ( + ExtraLogFiles, FormatOption, LogDateFormat, LogDir, - LogEmailNotifierType, - LogFileFormatsType, + LogEmailNotifier, + LogFileFormats, + LogFileFormatTypes, LogLevel, LogLevels, ) +# pylint: disable=too-many-instance-attributes @dataclass(frozen=True) class DefaultLoggingSettings: log_dir: LogDir = field(default_factory=lambda: os.path.join(os.getcwd(), "logs")) + log_dir_size_limit: int = 1024 # MB log_levels: LogLevels = field( default_factory=lambda: ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] ) log_date_format: LogDateFormat = "%Y-%m-%d %H:%M:%S" auto_initialization_enable: bool = True initialization_message_enable: bool = True - log_file_formats: LogFileFormatsType = field( + log_sql_queries_enable: bool = False + log_file_formats: LogFileFormats = field( default_factory=lambda: cast( - LogFileFormatsType, + LogFileFormats, { "DEBUG": 1, "INFO": 1, @@ -34,10 +39,35 @@ class DefaultLoggingSettings: }, ) ) + log_file_format_types: LogFileFormatTypes = field( + default_factory=lambda: cast( + LogFileFormatTypes, + { + "DEBUG": "normal", + "INFO": "normal", + "WARNING": "normal", + "ERROR": "normal", + "CRITICAL": "normal", + }, + ) + ) + + extra_log_files: ExtraLogFiles = field( + default_factory=lambda: cast( + ExtraLogFiles, + { + "DEBUG": False, + "INFO": False, + "WARNING": False, + "ERROR": False, + "CRITICAL": False, + }, + ) + ) - log_email_notifier: LogEmailNotifierType = field( + log_email_notifier: LogEmailNotifier = field( default_factory=lambda: cast( - LogEmailNotifierType, + LogEmailNotifier, { "ENABLE": False, "NOTIFY_ERROR": False, diff --git a/django_logging/constants/log_format_specifiers.py b/django_logging/constants/log_format_specifiers.py index 06d1972..78993ee 100644 --- a/django_logging/constants/log_format_specifiers.py +++ b/django_logging/constants/log_format_specifiers.py @@ -15,4 +15,5 @@ "thread", "threadName", "message", + "context", ] diff --git a/django_logging/settings/conf.py b/django_logging/settings/conf.py index 05185cc..3dae1ae 100644 --- a/django_logging/settings/conf.py +++ b/django_logging/settings/conf.py @@ -3,12 +3,19 @@ import os from typing import Dict, List, Optional -from django_logging.constants import FORMAT_OPTIONS, DefaultLoggingSettings +from django_logging.constants import ( + ALLOWED_EXTRA_FILE_TYPES, + ALLOWED_FILE_FORMAT_TYPES, + FORMAT_OPTIONS, + DefaultLoggingSettings, +) from django_logging.constants.config_types import ( + ExtraLogFiles, FormatOption, LogDateFormat, LogDir, - LogFileFormatsType, + LogFileFormats, + LogFileFormatTypes, LogLevel, LogLevels, NotifierLogLevels, @@ -30,7 +37,9 @@ def __init__( self, log_levels: LogLevels, log_dir: LogDir, - log_file_formats: LogFileFormatsType, + log_file_formats: LogFileFormats, + log_file_format_types: LogFileFormatTypes, + extra_log_files: ExtraLogFiles, console_level: LogLevel, console_format: FormatOption, colorize_console: bool, @@ -53,8 +62,10 @@ def __init__( self.email_notifier_log_format = self.resolve_format( log_email_notifier_log_format ) + self.log_file_format_types = log_file_format_types + self.extra_log_files = extra_log_files - def _resolve_file_formats(self, log_file_formats: LogFileFormatsType) -> Dict: + def _resolve_file_formats(self, log_file_formats: LogFileFormats) -> Dict: resolved_formats = {} for level in self.log_levels: format_option = log_file_formats.get(level, None) @@ -116,9 +127,22 @@ def __init__(self, log_config: LogConfig) -> None: def create_log_files(self) -> None: """Creates log files based on the log levels in the configuration.""" for log_level in self.log_config.log_levels: - log_file_path = os.path.join( - self.log_config.log_dir, f"{log_level.lower()}.log" - ) + fmt_type = self.log_config.log_file_format_types.get(log_level, "").lower() + extra_file = self.log_config.extra_log_files.get(log_level, False) + + if extra_file and fmt_type.upper() in ALLOWED_EXTRA_FILE_TYPES: + # Use separate files for extra file format structure + log_file_path = os.path.join( + self.log_config.log_dir, + fmt_type, + f"{log_level.lower()}.{fmt_type}", + ) + else: + # Use regular log file for normal, JSON, or XML + log_file_path = os.path.join( + self.log_config.log_dir, f"{log_level.lower()}.log" + ) + os.makedirs(os.path.dirname(log_file_path), exist_ok=True) if not os.path.exists(log_file_path): with open(log_file_path, "w", encoding="utf-8"): @@ -139,6 +163,7 @@ def get_log_file(self, log_level: LogLevel) -> Optional[str]: def set_conf(self) -> None: """Sets the logging configuration using the generated log files.""" + formatters = {} default_settings = DefaultLoggingSettings() handlers = { level.lower(): { @@ -146,7 +171,7 @@ def set_conf(self) -> None: "filename": log_file, "formatter": f"{level.lower()}", "level": level, - "filters": [level.lower()], + "filters": [level.lower(), "context_var_filter"], } for level, log_file in self.log_files.items() } @@ -154,13 +179,14 @@ def set_conf(self) -> None: "class": "logging.StreamHandler", "formatter": "console", "level": self.log_config.console_level, + "filters": ["context_var_filter"], } email_handler = { f"email_{level.lower()}": { "class": "django_logging.handlers.EmailHandler", "formatter": "email", "level": level, - "filters": [level.lower()], + "filters": [level.lower(), "context_var_filter"], } for level in self.log_config.email_notifier_log_levels if level @@ -176,13 +202,25 @@ def set_conf(self) -> None: for level in default_settings.log_levels } - formatters = { - level.lower(): { - "format": self.log_config.log_file_formats[level], - "datefmt": self.log_config.log_date_format, - } - for level in self.log_config.log_levels + # ContextVarFilter for context variables + filters["context_var_filter"] = { + "()": "django_logging.filters.ContextVarFilter", } + + for level in self.log_config.log_levels: + formatter = { + level.lower(): { + "format": self.log_config.log_file_formats[level], + "datefmt": self.log_config.log_date_format, + } + } + fmt_type = self.log_config.log_file_format_types.get(level, "None").upper() + if fmt_type in ALLOWED_FILE_FORMAT_TYPES: + formatter[level.lower()].update( + {"()": f"django_logging.formatters.{fmt_type}Formatter"} + ) + formatters.update(formatter) + formatters["console"] = { "format": self.log_config.console_format, "datefmt": self.log_config.log_date_format, diff --git a/django_logging/utils/get_conf.py b/django_logging/utils/get_conf.py index 0b962ce..3e2b014 100644 --- a/django_logging/utils/get_conf.py +++ b/django_logging/utils/get_conf.py @@ -18,6 +18,9 @@ def get_config(extra_info: bool = False) -> Dict: logging_defaults = DefaultLoggingSettings() console_defaults = DefaultConsoleSettings() + if not isinstance(log_settings, dict): + raise ValueError("DJANGO_LOGGING must be a dictionary with configs as keys") + log_levels = log_settings.get("LOG_FILE_LEVELS", logging_defaults.log_levels) log_dir = log_settings.get( "LOG_DIR", os.path.join(os.getcwd(), logging_defaults.log_dir) @@ -25,6 +28,12 @@ def get_config(extra_info: bool = False) -> Dict: log_file_formats = log_settings.get( "LOG_FILE_FORMATS", logging_defaults.log_file_formats ) + log_file_format_types = log_settings.get( + "LOG_FILE_FORMAT_TYPES", logging_defaults.log_file_format_types + ) + extra_log_files = log_settings.get( + "EXTRA_LOG_FILES", logging_defaults.extra_log_files + ) console_level = log_settings.get( "LOG_CONSOLE_LEVEL", console_defaults.log_console_level ) @@ -52,6 +61,8 @@ def get_config(extra_info: bool = False) -> Dict: "log_levels": log_levels, "log_dir": log_dir, "log_file_formats": log_file_formats, + "log_file_format_types": log_file_format_types, + "extra_log_files": extra_log_files, "console_level": console_level, "console_format": console_format, "colorize_console": colorize_console, @@ -61,7 +72,9 @@ def get_config(extra_info: bool = False) -> Dict: "log_email_notifier_log_format": log_email_notifier_log_format, } if extra_info: - config.update({"log_email_notifier": log_email_notifier}) + config.update( + {"log_email_notifier": log_email_notifier, "log_settings": log_settings} + ) return config @@ -115,3 +128,32 @@ def is_initialization_message_enabled() -> bool: return log_settings.get( "INITIALIZATION_MESSAGE_ENABLE", defaults.initialization_message_enable ) + + +def is_log_sql_queries_enabled() -> bool: + """Check if the LOG_SQL_QUERIES_ENABLE for the logging system is set to + True in Django settings. + + Returns: + bool: True if LOG_SQL_QUERIES_ENABLE, False otherwise. + Defaults to False if not specified. + + """ + log_settings = getattr(settings, "DJANGO_LOGGING", {}) + defaults = DefaultLoggingSettings() + + return log_settings.get("LOG_SQL_QUERIES_ENABLE", defaults.log_sql_queries_enable) + + +def get_log_dir_size_limit() -> int: + """Check for the LOG_DIR_SIZE_LIMIT for managing the log dir size. + + Returns: + int: the limit of log directory size. + Defaults to 1024 MB if not specified. + + """ + log_settings = getattr(settings, "DJANGO_LOGGING", {}) + defaults = DefaultLoggingSettings() + + return log_settings.get("LOG_DIR_SIZE_LIMIT", defaults.log_dir_size_limit) diff --git a/django_logging/utils/set_conf.py b/django_logging/utils/set_conf.py index eedc36d..1530285 100644 --- a/django_logging/utils/set_conf.py +++ b/django_logging/utils/set_conf.py @@ -4,10 +4,12 @@ from django_logging.constants.ansi_colors import AnsiColors from django_logging.constants.config_types import ( + ExtraLogFiles, FormatOption, LogDateFormat, LogDir, - LogFileFormatsType, + LogFileFormats, + LogFileFormatTypes, LogLevel, LogLevels, NotifierLogLevels, @@ -23,7 +25,9 @@ def set_config( log_levels: LogLevels, log_dir: LogDir, - log_file_formats: LogFileFormatsType, + log_file_formats: LogFileFormats, + log_file_format_types: LogFileFormatTypes, + extra_log_files: ExtraLogFiles, console_level: LogLevel, console_format: FormatOption, colorize_console: bool, @@ -42,6 +46,8 @@ def set_config( log_levels (LogLevels): A list specifying the log levels for different handlers. log_dir (LogDir): The directory where log files will be stored. log_file_formats (LogFileFormatsType): The format of the log files. + log_file_format_types (LogFileFormatTypes): The type of the log format of the log files. + extra_log_files (ExtraLogFiles): Whether to create separate files for custom format types. console_level (LogLevel): The log level for console output. console_format (FormatOption): The format for console log messages. colorize_console (bool): Whether to colorize console output. @@ -61,6 +67,8 @@ def set_config( ... log_levels=['DEBUG', 'INFO'], ... log_dir='/var/log/myapp/', ... log_file_formats={'INFO': '%(levelname)s %(asctime)s %(message)s'}, + ... log_file_format_types={'INFO': 'JSON'}, + ... extra_log_files={'INFO': True}, ... console_level='DEBUG', ... console_format='{message}', ... colorize_console=True, @@ -83,6 +91,8 @@ def set_config( log_levels, log_dir, log_file_formats, + log_file_format_types, + extra_log_files, console_level, console_format, colorize_console, @@ -121,13 +131,13 @@ def set_config( logger = getLogger(__name__) logger.info( - "Logging initialized with the following configurations:\n" - "Log File levels: %s.\n" - "Log files are being written to: %s.\n" - "Console output level: %s.\n" - "Colorize console: %s.\n" - "Log date format: %s.\n" - "Email notifier enabled: %s.\n", + "Logging initialized with the following configurations:" + "\n\tLog File levels: %s." + "\n\tLog files are being written to: %s." + "\n\tConsole output level: %s." + "\n\tColorize console: %s." + "\n\tLog date format: %s." + "\n\tEmail notifier enabled: %s.", log_levels, log_dir, console_level or "default (DEBUG)", diff --git a/django_logging/validators/config_validators.py b/django_logging/validators/config_validators.py index 67de299..8448708 100644 --- a/django_logging/validators/config_validators.py +++ b/django_logging/validators/config_validators.py @@ -1,6 +1,6 @@ import os import re -from typing import List +from typing import Dict, List from django.core.checks import Error @@ -11,7 +11,7 @@ ) from django_logging.constants.config_types import ( FormatOption, - LogEmailNotifierType, + LogEmailNotifier, LogLevels, ) @@ -206,7 +206,7 @@ def parse_format_string(format_string: str) -> set: return errors -def validate_email_notifier(notifier_config: LogEmailNotifierType) -> List[Error]: +def validate_email_notifier(notifier_config: LogEmailNotifier) -> List[Error]: errors = [] if not isinstance(notifier_config, dict): errors.append( @@ -246,7 +246,7 @@ def validate_integer_setting(value: int, config_name: str) -> List[Error]: Error( f"{config_name} is not a valid integer.", hint=f"Ensure {config_name} is a valid positive integer", - id=f"django_logging.E019_{config_name}", + id=f"django_logging.E021_{config_name}", ) ) return errors diff --git a/django_logging/validators/email_settings_validator.py b/django_logging/validators/email_settings_validator.py index 8891678..8eb849a 100644 --- a/django_logging/validators/email_settings_validator.py +++ b/django_logging/validators/email_settings_validator.py @@ -31,7 +31,7 @@ def check_email_settings(require_admin_email: bool = True) -> List[Error]: errors.append( Error( f"Missing required email settings: {missing}", - hint="Email settings required because you set LOG_EMAIL_NOTIFIER['ENABLE'] to True,\n" + hint="Email settings required because you trying to send an email," "Ensure all required email settings are properly configured in your settings file.", id="django_logging.E021", )