From bfa140ccc453be4df0eff69b2a7859b765b16de8 Mon Sep 17 00:00:00 2001 From: Shashank Reddy Boyapally Date: Mon, 29 Jul 2024 15:55:48 -0400 Subject: [PATCH] adding templating through env vars for cmd mode Signed-off-by: Shashank Reddy Boyapally --- orion.py | 2 +- pkg/config.py | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ pkg/daemon.py | 32 ++++++-------------------- pkg/utils.py | 24 ------------------- 4 files changed, 72 insertions(+), 50 deletions(-) create mode 100644 pkg/config.py diff --git a/orion.py b/orion.py index dba5f40..eb73488 100644 --- a/orion.py +++ b/orion.py @@ -11,7 +11,7 @@ import uvicorn from fmatch.logrus import SingletonLogger from pkg.runTest import run -from pkg.utils import load_config +from pkg.config import load_config import pkg.constants as cnsts warnings.filterwarnings("ignore", message="Unverified HTTPS request.*") diff --git a/pkg/config.py b/pkg/config.py new file mode 100644 index 0000000..74dbf8f --- /dev/null +++ b/pkg/config.py @@ -0,0 +1,64 @@ +# pylint: disable = line-too-long +""" +Module file for config reading and loading +""" + + +import os +import sys +from typing import Any, Dict, Set + +from jinja2 import Environment, Template, meta +from fmatch.logrus import SingletonLogger +import yaml + + +def load_config(config: str, parameters: Dict= None) -> Dict[str, Any]: + """Loads config file + + Args: + config (str): path to config file + logger (Logger): logger + + Returns: + dict: dictionary of the config file + """ + env_vars = {k.lower(): v for k, v in os.environ.items()} + merged_parameters = {} + merged_parameters.update(env_vars) + if parameters: + merged_parameters.update(parameters) + logger_instance = SingletonLogger.getLogger("Orion") + try: + with open(config, "r", encoding="utf-8") as template_file: + template_content = template_file.read() + logger_instance.debug("The %s file has successfully loaded", config) + except FileNotFoundError as e: + logger_instance.error("Config file not found: %s", e) + sys.exit(1) + except Exception as e: # pylint: disable=broad-exception-caught + logger_instance.error("An error occurred: %s", e) + sys.exit(1) + + required_parameters = get_template_variables(template_content) + logger_instance.debug(f"Variables required by the template: {required_parameters}") + + # Check for missing variables + missing_vars = required_parameters - merged_parameters.keys() + if missing_vars: + logger_instance.error(f"Missing required parameters: {missing_vars}, use environment variables to set") + sys.exit(1) + + template = Template(template_content) + rendered_config_yaml = template.render(merged_parameters) + rendered_config = yaml.safe_load(rendered_config_yaml) + print(rendered_config) + return rendered_config + + +def get_template_variables(template_content: str) -> Set[str]: + """Extracts all variables from the Jinja2 template content.""" + env = Environment() + parsed_content = env.parse(template_content) + variables = meta.find_undeclared_variables(parsed_content) + return variables diff --git a/pkg/daemon.py b/pkg/daemon.py index ea6d522..cf9366f 100644 --- a/pkg/daemon.py +++ b/pkg/daemon.py @@ -6,10 +6,9 @@ import os from typing import Any from fastapi import FastAPI, HTTPException -from jinja2 import Template import pkg_resources -import yaml from fmatch.logrus import SingletonLogger +from pkg.config import load_config import pkg.constants as cnsts from . import runTest @@ -37,7 +36,8 @@ async def daemon_changepoint( # pylint: disable = R0913 json: json object of the changepoints and metrics """ parameters = {"version": version} - config_file_name=test_name+".yml" + config_file_name=f"{test_name}.yml" + config_path = pkg_resources.resource_filename("configs", config_file_name) option_arguments = { "config": config_file_name, "save_data_path": "output.csv", @@ -47,7 +47,7 @@ async def daemon_changepoint( # pylint: disable = R0913 "uuid": uuid, "lookback":lookback, "baseline": baseline, - "configMap": render_template(config_file_name, parameters), + "configMap": load_config(config_path, parameters), "convert_tinyurl": convert_tinyurl.lower() not in "false", } filter_changepoints = ( @@ -90,7 +90,7 @@ async def get_options() -> Any: raise HTTPException(status_code=500, detail=str(e)) from e @app.get("/daemon/anomaly") -async def daemon_anomaly( # pylint: disable = R0913 +async def daemon_anomaly( # pylint: disable = R0913, R0914 version: str = "4.17", uuid: str = "", baseline: str = "", @@ -111,6 +111,7 @@ async def daemon_anomaly( # pylint: disable = R0913 """ parameters = {"version": version} config_file_name=test_name+".yml" + config_path = pkg_resources.resource_filename("configs", config_file_name) option_arguments = { "config": config_file_name, "save_data_path": "output.csv", @@ -120,7 +121,7 @@ async def daemon_anomaly( # pylint: disable = R0913 "uuid": uuid, "lookback":lookback, "baseline": baseline, - "configMap": render_template(config_file_name, parameters), + "configMap": load_config(config_path, parameters), "anomaly_window": int(anomaly_window), "min_anomaly_percent":int(min_anomaly_percent), "convert_tinyurl": convert_tinyurl.lower() not in "false", @@ -136,22 +137,3 @@ async def daemon_anomaly( # pylint: disable = R0913 for key, value in result.items(): result[key] = list(filter(lambda x: x.get("is_changepoint", False), value)) return result - - -def render_template(test_name: str, parameters: dict[str,Any]) -> Any: - """replace parameters in the config file - - Args: - file_name (str): the config file - parameters (dict): parameters to be replaces - - Returns: - dict: configMap in dict - """ - config_path = pkg_resources.resource_filename("configs", test_name) - with open(config_path, "r", encoding="utf-8") as template_file: - template_content = template_file.read() - template = Template(template_content) - rendered_config_yaml = template.render(parameters) - rendered_config = yaml.safe_load(rendered_config_yaml) - return rendered_config diff --git a/pkg/utils.py b/pkg/utils.py index 698f9bb..b6b8710 100644 --- a/pkg/utils.py +++ b/pkg/utils.py @@ -16,10 +16,7 @@ from tabulate import tabulate from fmatch.matcher import Matcher from fmatch.logrus import SingletonLogger - -import yaml import pandas as pd - import pyshorteners @@ -132,28 +129,7 @@ def extract_metadata_from_test(test: Dict[str, Any]) -> Dict[Any, Any]: return metadata -def load_config(config: str) -> Dict[str, Any]: - """Loads config file - - Args: - config (str): path to config file - logger (Logger): logger - Returns: - dict: dictionary of the config file - """ - logger_instance = SingletonLogger.getLogger("Orion") - try: - with open(config, "r", encoding="utf-8") as file: - data = yaml.safe_load(file) - logger_instance.debug("The %s file has successfully loaded", config) - except FileNotFoundError as e: - logger_instance.error("Config file not found: %s", e) - sys.exit(1) - except Exception as e: # pylint: disable=broad-exception-caught - logger_instance.error("An error occurred: %s", e) - sys.exit(1) - return data def get_datasource(data: Dict[Any, Any]) -> str: