From d38d3e4c9b54b77b14583509c7db3c4e6cbdc410 Mon Sep 17 00:00:00 2001 From: MEHRSHAD MIRSHEKARY Date: Fri, 4 Oct 2024 16:07:17 +0330 Subject: [PATCH 1/2] :sparkles: feat(command): Add Django Management Command for Generating Pretty JSON from Log Directory #### Key Features: - **Command Overview**: The command is designed to search for JSON files, handle multiple JSON objects within a single file, and output a prettified JSON format, enhancing readability and usability for developers and maintainers. - **Directory Setup**: Utilizes a helper function to establish the required directory structure, ensuring the presence of both the original and formatted directories. - **File Processing**: The command processes each .json file found in the specified directory, incrementally reading and reformatting the content to ensure that all JSON objects are valid and properly structured. - **Error Handling**: Implements error handling for file access issues and reports any incomplete JSON objects encountered during the processing, making it easier to troubleshoot formatting problems. - **User Feedback**: Provides console feedback during execution, informing users of the status of file processing and any issues that may arise. This management command simplifies the task of formatting JSON logs, making them more accessible for analysis and further processing. --- .../commands/generate_pretty_json.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 django_logging/management/commands/generate_pretty_json.py diff --git a/django_logging/management/commands/generate_pretty_json.py b/django_logging/management/commands/generate_pretty_json.py new file mode 100644 index 0000000..d98d8b5 --- /dev/null +++ b/django_logging/management/commands/generate_pretty_json.py @@ -0,0 +1,103 @@ +import json +import os +from typing import Dict, Tuple + +from django.conf import settings +from django.core.management.base import BaseCommand + +from django_logging.constants import DefaultLoggingSettings +from django_logging.constants.config_types import LogDir +from django_logging.utils.command.process_file import process_files, setup_directories + + +class Command(BaseCommand): + """A Django management command to find JSON files within a specified log + directory and generate pretty JSON. + + The command looks for `.json` files inside the `json` subdirectory of the log directory, attempts to + parse multiple JSON objects from a single file, and then formats them into a valid JSON array. + + The reformatted JSON content is saved in a `pretty` subdirectory with the prefix `formatted_`. + + """ + + help = "Find JSON files in log directory and generates pretty JSON" + + def handle(self, *args: Tuple, **kwargs: Dict) -> None: + """Main command handler. This method retrieves the log directory, sets + up necessary directories, processes each `.json` file, and reformats + the content. + + Args: + *args: Additional positional arguments (not used). + **kwargs: Additional keyword arguments (not used). + + """ + default_settings = DefaultLoggingSettings() + log_dir: LogDir = settings.DJANGO_LOGGING.get( + "LOG_DIR", os.path.join(os.getcwd(), default_settings.log_dir) + ) + + try: + json_dir, pretty_dir = setup_directories(log_dir, "json") + except FileNotFoundError as e: + self.stdout.write(self.style.ERROR(str(e))) + return + + for file_path, filename in process_files(json_dir, ".json", self.reformat_json): + self.stdout.write(self.style.NOTICE(f"Processing file: {file_path}")) + + new_file_path: str = os.path.join(pretty_dir, f"formatted_{filename}") + self.reformat_json(file_path, new_file_path) + + self.stdout.write( + self.style.SUCCESS( + f"File {filename} reformatted and generated new pretty file successfully." + ) + ) + + def reformat_json(self, file_path: str, new_file_path: str) -> None: + """Parses multiple JSON objects from a file incrementally and writes + them to a new file as a valid JSON array. + + Args: + file_path (str): The path to the original JSON file. + new_file_path (str): The path where the reformatted JSON file will be saved. + + """ + with open(file_path, encoding="utf-8") as infile, open( + new_file_path, "w", encoding="utf-8" + ) as outfile: + outfile.write("[\n") # Start the JSON array + first_object = True # Flag to handle commas + + buffer = "" # This will accumulate the JSON content + + for line in infile: + line = line.strip() + + if not line: + continue # Skip empty lines + + buffer += line + + # Try to parse the current buffer as a complete JSON object + try: + json_object = json.loads(buffer) + json_line = json.dumps(json_object, indent=4) + + if not first_object: + outfile.write(",\n") # Add a comma before subsequent objects + outfile.write(json_line) + + first_object = False + buffer = "" # Clear the buffer after successful parsing + except json.JSONDecodeError: + # Keep accumulating if it's not a complete JSON object yet + continue + + if buffer: + # If any partial JSON is left in the buffer, log an error + self.stdout.write(self.style.ERROR(f"Incomplete JSON object: {buffer}")) + + outfile.write("\n]") # End the JSON array From 980f28d8a50a9a8aa5229c68f9bef4fad30fb690 Mon Sep 17 00:00:00 2001 From: MEHRSHAD MIRSHEKARY Date: Fri, 4 Oct 2024 16:10:36 +0330 Subject: [PATCH 2/2] :sparkles: feat(commands): Add Django Management Command for Reformatting XML Files #### Key Features: - **Command Overview**: The command searches for .xml files in the specified log directory and reformats them by encapsulating the existing content within a element, creating a valid XML structure. - **Directory Setup**: Utilizes helper functions to ensure that the required input and output directories are properly set up, enhancing usability and preventing file access errors. - **File Processing**: Iterates through each XML file, applies the reformatting logic, and saves the modified content to a new directory, helping maintain the original files unaltered. - **Error Handling**: Implements error handling to manage file access issues gracefully, providing feedback to users about any encountered problems. - **User Feedback**: Offers real-time console output during execution, notifying users of the processing status for each file and confirming successful reformatting. This management command simplifies the task of restructuring XML logs, making them more organized and easier to work with for further analysis or integration. Closes #94 --- .../commands/generate_pretty_xml.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 django_logging/management/commands/generate_pretty_xml.py diff --git a/django_logging/management/commands/generate_pretty_xml.py b/django_logging/management/commands/generate_pretty_xml.py new file mode 100644 index 0000000..7adc729 --- /dev/null +++ b/django_logging/management/commands/generate_pretty_xml.py @@ -0,0 +1,77 @@ +import os +from typing import Any, Dict, Tuple + +from django.conf import settings +from django.core.management.base import BaseCommand + +from django_logging.constants import DefaultLoggingSettings +from django_logging.constants.config_types import LogDir +from django_logging.utils.command.process_file import process_files, setup_directories + + +class Command(BaseCommand): + """Command to find and reformat XML files in a specified directory. + + This command processes all XML files in the specified log directory, reformats + them by wrapping their content in a element, and saves the reformatted + files to a new directory. It handles parsing errors and logs the process steps. + + Attributes: + help (str): A brief description of the command's functionality. + + """ + + help = "Find and reformat XML files in a directory" + + def handle(self, *args: Tuple[Any], **kwargs: Dict[str, Any]) -> None: + """Handles the command execution. + + Args: + *args: Positional arguments passed to the command. + **kwargs: Keyword arguments passed to the command. + + """ + default_settings = DefaultLoggingSettings() + log_dir: LogDir = settings.DJANGO_LOGGING.get( + "LOG_DIR", os.path.join(os.getcwd(), default_settings.log_dir) + ) + + try: + xml_dir, pretty_dir = setup_directories(log_dir, "xml") + except FileNotFoundError as e: + self.stdout.write(self.style.ERROR(str(e))) + return + + for file_path, filename in process_files( + xml_dir, ".xml", self.reformat_and_write_xml + ): + self.stdout.write(self.style.NOTICE(f"Processing file: {file_path}")) + + new_file_path = os.path.join(pretty_dir, f"formatted_{filename}") + self.reformat_and_write_xml(file_path, new_file_path) + self.stdout.write( + self.style.SUCCESS(f"File {filename} reformatted successfully.") + ) + + def reformat_and_write_xml(self, file_path: str, new_file_path: str) -> None: + """Reformats XML content by wrapping it in a element and writes + it to a new file. + + Args: + file_path (str): The path of the original XML file to be reformatted. + new_file_path (str): The path where the reformatted XML file will be saved. + + """ + with open(file_path, encoding="utf-8") as infile, open( + new_file_path, "w", encoding="utf-8" + ) as outfile: + # Start the element + outfile.write("\n") + + for line in infile: + # Write each line to the formatted file + if line.strip(): # Only process non-empty lines + outfile.write(line) + + # End the element + outfile.write("\n")