Add Folder Handling and Logging Enhancements (#12)

#### Overview
This PR introduces folder structure support in the `generate` and
`validate` commands, along with improved logging functionality using
`colorlog` for better debugging and runtime insights.

#### Changes
- **Folder Structure Support**:
- The `generate.py` script now supports creating folder structures from
YAML definitions.
- Added validation for the folder structure in `validate.py`, ensuring
folders are correctly defined and contain a valid `struct` key.
- **Logging Enhancements**:
- New logging configuration file `logging_config.py` introduced,
integrating `colorlog` for color-coded log output.
- Refactored log levels in various parts of the code, changing some
`info` logs to `debug` where appropriate for better verbosity control.
- **Dependency Update**:
  - Added `colorlog` to `requirements.txt` to support colored logging.

#### Justification
The ability to define folder structures in YAML is critical for handling
more complex project setups, especially for projects relying on modular
directory layouts. Enhanced logging is essential for improving the
traceability and debuggability of the tool's operations.

#### Impact
- Teams will now be able to define and validate folder structures more
easily within the configuration files.
- Logging improvements will make debugging easier and more informative,
particularly during dry runs and when setting up folder hierarchies.
- The addition of `colorlog` may require updating environments to
include the new dependency.

requirements.txt

@@ -5,3 +5,4 @@ python-dotenv
 jinja2
 PyGithub
 argcomplete
+colorlog

struct_module/commands/generate.py

@@ -1,6 +1,7 @@
 from struct_module.commands import Command
 import os
 import yaml
+import argparse
 from struct_module.file_item import FileItem
 from struct_module.completers import file_strategy_completer
 
@@ -32,6 +33,8 @@ def execute(self, args):
 
 
   def _create_structure(self, args):
+    if isinstance(args, dict):
+        args = argparse.Namespace(**args)
     if args.structure_definition.startswith("file://") and args.structure_definition.endswith(".yaml"):
       with open(args.structure_definition[7:], 'r') as f:
         config = yaml.safe_load(f)
@@ -46,6 +49,7 @@ def _create_structure(self, args):
 
     template_vars = dict(item.split('=') for item in args.vars.split(',')) if args.vars else None
     config_structure = config.get('structure', [])
+    config_folders = config.get('folders', [])
     config_variables = config.get('variables', [])
 
     for item in config_structure:
@@ -75,3 +79,29 @@ def _create_structure(self, args):
           args.backup or None,
           args.file_strategy or 'overwrite'
         )
+
+    for item in config_folders:
+      for folder, content in item.items():
+        folder_path = os.path.join(args.base_path, folder)
+        if args.dry_run:
+          self.logger.info(f"[DRY RUN] Would create folder: {folder_path}")
+          continue
+        os.makedirs(folder_path, exist_ok=True)
+        self.logger.info(f"Created folder: {folder_path}")
+
+        # check if content has struct value
+        if 'struct' in content:
+          self.logger.info(f"Generating structure in folder: {folder} with struct {content['struct']}")
+          self._create_structure({
+            'structure_definition': content['struct'],
+            'base_path': folder_path,
+            'structures_path': args.structures_path,
+            'dry_run': args.dry_run,
+            'vars': args.vars,
+            'backup': args.backup,
+            'file_strategy': args.file_strategy,
+            'global_system_prompt': args.global_system_prompt,
+          })
+          self.logger.info(f"Generated structure in folder: {folder} with struct {content['struct']}")
+        else:
+          self.logger.warning(f"Unsupported content in folder: {folder}")

struct_module/commands/validate.py

@@ -19,9 +19,37 @@ def execute(self, args):
         config = yaml.safe_load(f)
 
       self._validate_structure_config(config.get('structure', []))
+      self._validate_folders_config(config.get('folders', []))
       self._validate_variables_config(config.get('variables', []))
 
 
+    # Validate the 'folders' key in the configuration file
+    # folders should be defined as a list of dictionaries
+    # each dictionary should have a 'struct' key
+    #
+    # Example:
+    # folders:
+    #   - .devops/modules/my_module_one:
+    #       struct: terraform-module
+    #   - .devops/modules/my_module_two:
+    #       struct: terraform-module
+    def _validate_folders_config(self, folders):
+      if not isinstance(folders, list):
+        raise ValueError("The 'folders' key must be a list.")
+      for item in folders:
+        if not isinstance(item, dict):
+            raise ValueError("Each item in the 'folders' list must be a dictionary.")
+        for name, content in item.items():
+            if not isinstance(name, str):
+              raise ValueError("Each name in the 'folders' item must be a string.")
+            if not isinstance(content, dict):
+              raise ValueError(f"The content of '{name}' must be a dictionary.")
+            if 'struct' not in content:
+              raise ValueError(f"Dictionary item '{name}' must contain a 'struct' key.")
+            if not isinstance(content['struct'], str):
+              raise ValueError(f"The 'struct' value for '{name}' must be a string.")
+
+
     # Validate the 'variables' key in the configuration file
     # variables should be defined as a list of dictionaries
     # each dictionary should have a 'name' key and optionall 'default' value

struct_module/file_item.py

@@ -35,7 +35,7 @@ def __init__(self, properties):
     def _configure_openai(self):
       self.openai_client = OpenAI(api_key=openai_api_key)
       if not openai_model:
-        self.logger.info("OpenAI model not found. Using default model.")
+        self.logger.debug("OpenAI model not found. Using default model.")
         self.openai_model = "gpt-3.5-turbo"
       else:
         self.logger.debug(f"Using OpenAI model: {openai_model}")
@@ -144,7 +144,7 @@ def create(self, base_path, dry_run=False, backup_path=None, file_strategy='over
 
       with open(file_path, 'w') as f:
         f.write(self.content)
-      self.logger.info(f"Created file: {file_path} with content: \n\n{self.content}")
+      self.logger.debug(f"Created file: {file_path} with content: \n\n{self.content}")
 
       if self.permissions:
         os.chmod(file_path, int(self.permissions, 8))

struct_module/logging_config.py

@@ -0,0 +1,31 @@
+# FILE: struct_module/logging_config.py
+import logging
+import colorlog
+
+def configure_logging(level=logging.INFO, log_file=None):
+    """Configure logging with colorlog."""
+    handler = colorlog.StreamHandler()
+    handler.setFormatter(colorlog.ColoredFormatter(
+        "%(log_color)s[%(asctime)s][%(levelname)s][struct] >>> %(message)s",
+        datefmt='%Y-%m-%d %H:%M:%S',
+        log_colors={
+            'DEBUG': 'cyan',
+            'INFO': 'green',
+            'WARNING': 'yellow',
+            'ERROR': 'red',
+            'CRITICAL': 'bold_red',
+        }
+    ))
+
+    logging.basicConfig(
+        level=level,
+        handlers=[handler],
+    )
+
+    if log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setFormatter(logging.Formatter(
+            "[%(asctime)s][%(levelname)s][struct] >>> %(message)s",
+            datefmt='%Y-%m-%d %H:%M:%S'
+        ))
+        logging.getLogger().addHandler(file_handler)

struct_module/main.py

@@ -6,6 +6,7 @@
 from struct_module.commands.info import InfoCommand
 from struct_module.commands.validate import ValidateCommand
 from struct_module.commands.list import ListCommand
+from struct_module.logging_config import configure_logging
 
 
 
@@ -42,11 +43,8 @@ def main():
 
     logging_level = getattr(logging, args.log.upper(), logging.INFO)
 
-    logging.basicConfig(
-      level=logging_level,
-      filename=args.log_file,
-      format='[%(asctime)s][%(levelname)s][struct] >>> %(message)s',
-    )
+    configure_logging(level=logging_level, log_file=args.log_file)
+
 
     args.func(args)