From 8966d358b9c08c2ff05bfeb85ac926aa70e267f5 Mon Sep 17 00:00:00 2001 From: Shashank Reddy Boyapally Date: Wed, 20 Mar 2024 23:41:32 -0400 Subject: [PATCH] fixed logging and added uuid comparison Signed-off-by: Shashank Reddy Boyapally --- orion.py | 15 ++++---- pkg/daemon.py | 15 ++++++-- pkg/logrus.py | 11 +++--- pkg/runTest.py | 34 +++++++++--------- pkg/utils.py | 98 +++++++++++++++++++++++++++++++++++++++++--------- 5 files changed, 127 insertions(+), 46 deletions(-) diff --git a/orion.py b/orion.py index 13f95b8..998eec1 100644 --- a/orion.py +++ b/orion.py @@ -10,7 +10,6 @@ from pkg.logrus import SingletonLogger from pkg.runTest import run -logger_instance = SingletonLogger(debug=False).logger @click.group() @@ -35,7 +34,9 @@ def cli(max_content_width=120): # pylint: disable=unused-argument default="text", help="Choose output format (json or text)", ) -def cmd_analysis(config, debug, output_path, hunter_analyze, output_format): +@click.option("--uuid", default="", help="UUID to use as base for comparisons") +@click.option("--baseline", default="", help="Baseline UUID(s) to to compare against uuid") +def cmd_analysis(**kwargs): """ Orion runs on command line mode, and helps in detecting regressions @@ -45,13 +46,13 @@ def cmd_analysis(config, debug, output_path, hunter_analyze, output_format): debug (bool): lets you log debug mode output (str): path to the output csv file """ - level = logging.DEBUG if debug else logging.INFO - logger_instance.setLevel(level) + level = logging.DEBUG if kwargs['debug'] else logging.INFO + logger_instance = SingletonLogger(debug=level).logger logger_instance.info("🏹 Starting Orion in command-line mode") - output = run(config, output_path, hunter_analyze, output_format) + output = run(**kwargs) for test_name, result_table in output.items(): print(test_name) - print("-"*len(test_name)) + print("="*len(test_name)) print(result_table) @@ -63,7 +64,7 @@ def rundaemon(debug): \b """ level = logging.DEBUG if debug else logging.INFO - logger_instance.setLevel(level) + logger_instance = SingletonLogger(debug=level).logger logger_instance.info("🏹 Starting Orion in Daemon mode") uvicorn.run("pkg.daemon:app", port=8000) diff --git a/pkg/daemon.py b/pkg/daemon.py index d471066..0c2ad4d 100644 --- a/pkg/daemon.py +++ b/pkg/daemon.py @@ -1,6 +1,7 @@ """ Module to run orion in daemon mode """ +import logging import shutil import os @@ -10,11 +11,11 @@ from . import runTest app = FastAPI() -logger_instance = SingletonLogger(debug=False).logger +logger_instance = SingletonLogger(debug=logging.INFO).logger @app.post("/daemon") -async def daemon(file: UploadFile = File(...)): +async def daemon(file: UploadFile = File(...), uuid: str = "", baseline: str = ""): """starts listening on port 8000 on url /daemon Args: @@ -27,7 +28,15 @@ async def daemon(file: UploadFile = File(...)): new_file_name = f"{file_name}_copy{file_extension}" with open(new_file_name, "wb") as buffer: shutil.copyfileobj(file.file, buffer) - result = runTest.run(new_file_name, "output.csv", True, "json") + argDict={ + 'config': new_file_name, + 'output_path': "output.csv", + 'hunter_analyze': True, + 'output_format': "json", + 'uuid':uuid, + 'baseline':baseline, + } + result = runTest.run(**argDict) try: os.remove(new_file_name) except OSError as e: diff --git a/pkg/logrus.py b/pkg/logrus.py index 2f300fb..d9c9539 100644 --- a/pkg/logrus.py +++ b/pkg/logrus.py @@ -1,18 +1,21 @@ """ Logger for orion """ + import logging import sys + class SingletonLogger: """Singleton logger to set logging at one single place Returns: _type_: _description_ """ + _instance = None - def __new__(cls, debug=False): + def __new__(cls, debug): if cls._instance is None: cls._instance = super().__new__(cls) cls._instance._logger = cls._initialize_logger(debug) @@ -20,13 +23,13 @@ def __new__(cls, debug=False): @staticmethod def _initialize_logger(debug): - level = logging.DEBUG if debug else logging.INFO + level = debug # if debug else logging.INFO logger = logging.getLogger("Orion") logger.setLevel(level) handler = logging.StreamHandler(sys.stdout) handler.setLevel(level) formatter = logging.Formatter( - "%(asctime)s - %(filename)s-%(lineno)d - %(name)s - %(levelname)s - %(message)s" + "%(asctime)s - %(name)s - %(levelname)s - file: %(filename)s - line: %(lineno)d - %(message)s" # pylint: disable = line-too-long ) handler.setFormatter(formatter) logger.addHandler(handler) @@ -39,4 +42,4 @@ def logger(self): Returns: _type_: _description_ """ - return self._logger # pylint: disable = no-member + return self._logger # pylint: disable = no-member diff --git a/pkg/runTest.py b/pkg/runTest.py index 98ba007..c355f59 100644 --- a/pkg/runTest.py +++ b/pkg/runTest.py @@ -1,23 +1,18 @@ """ run test """ + +import logging from fmatch.matcher import Matcher from pkg.logrus import SingletonLogger -from pkg.utils import ( - run_hunter_analyze, - load_config, - get_es_url, - process_test -) +from pkg.utils import run_hunter_analyze, load_config, get_es_url, process_test -logger_instance= SingletonLogger().logger -def run(config, output_path, hunter_analyze,output_format): +def run(**kwargs): """run method to start the tests Args: config (_type_): file path to config file - debug (_type_): debug to be true or false output_path (_type_): output path to save the data hunter_analyze (_type_): changepoint detection through hunter. defaults to True output_format (_type_): output to be table or json @@ -25,13 +20,20 @@ def run(config, output_path, hunter_analyze,output_format): Returns: _type_: _description_ """ - data = load_config(config, logger_instance) - ES_URL = get_es_url(data,logger=logger_instance) + logger_instance = SingletonLogger(debug=logging.INFO).logger + data = load_config(kwargs["config"]) + ES_URL = get_es_url(data) result_output = {} for test in data["tests"]: - match = Matcher(index="perf_scale_ci",level=logger_instance.level, ES_URL=ES_URL) - result = process_test(test, match, logger_instance, output_path) - if hunter_analyze: - testname,result_data=run_hunter_analyze(result, test,output=output_format,matcher=match) - result_output[testname]=result_data + match = Matcher( + index="perf_scale_ci", level=logger_instance.level, ES_URL=ES_URL + ) + result = process_test( + test, match, kwargs["output_path"], kwargs["uuid"], kwargs["baseline"] + ) + if kwargs["hunter_analyze"]: + testname, result_data = run_hunter_analyze( + result, test, output=kwargs["output_format"], matcher=match + ) + result_output[testname] = result_data return result_output diff --git a/pkg/utils.py b/pkg/utils.py index 9169b93..778bb36 100644 --- a/pkg/utils.py +++ b/pkg/utils.py @@ -6,7 +6,9 @@ from functools import reduce import json +import logging import os +import re import sys import yaml @@ -15,6 +17,11 @@ from hunter.report import Report, ReportType from hunter.series import Metric, Series +from pkg.logrus import SingletonLogger + + + + def run_hunter_analyze(merged_df, test, output, matcher): """Start hunter analyze function @@ -71,7 +78,6 @@ def parse_json_output(merged_df, change_points_by_metric,matcher): Returns: _type_: _description_ """ - df_json = merged_df.to_json(orient="records") df_json = json.loads(df_json) @@ -99,7 +105,7 @@ def parse_json_output(merged_df, change_points_by_metric,matcher): # pylint: disable=too-many-locals -def get_metric_data(ids, index, metrics, match, logger): +def get_metric_data(ids, index, metrics, match): """Gets details metrics basked on metric yaml list Args: @@ -112,6 +118,7 @@ def get_metric_data(ids, index, metrics, match, logger): Returns: dataframe_list: dataframe of the all metrics """ + logger= SingletonLogger(debug=logging.INFO).logger dataframe_list = [] for metric in metrics: metric_name = metric["name"] @@ -152,7 +159,7 @@ def get_metric_data(ids, index, metrics, match, logger): return dataframe_list -def get_metadata(test, logger): +def get_metadata(test): """Gets metadata of the run from each test Args: @@ -161,13 +168,14 @@ def get_metadata(test, logger): Returns: dict: dictionary of the metadata """ + logger= SingletonLogger(debug=logging.INFO).logger metadata = test["metadata"] metadata["ocpVersion"] = str(metadata["ocpVersion"]) logger.debug("metadata" + str(metadata)) return metadata -def load_config(config, logger): +def load_config(config): """Loads config file Args: @@ -177,6 +185,7 @@ def load_config(config, logger): Returns: dict: dictionary of the config file """ + logger= SingletonLogger(debug=logging.INFO).logger try: with open(config, "r", encoding="utf-8") as file: data = yaml.safe_load(file) @@ -190,7 +199,7 @@ def load_config(config, logger): return data -def get_es_url(data, logger): +def get_es_url(data): """Gets es url from config or env Args: @@ -200,6 +209,7 @@ def get_es_url(data, logger): Returns: str: es url """ + logger= SingletonLogger(debug=logging.INFO).logger if "ES_SERVER" in data.keys(): return data["ES_SERVER"] if "ES_SERVER" in os.environ: @@ -208,7 +218,7 @@ def get_es_url(data, logger): sys.exit(1) -def get_index_and_ids(metadata, uuids, match): +def get_index_and_ids(metadata, uuids, match, baseline): """returns the index to be used and runs as uuids Args: @@ -224,11 +234,15 @@ def get_index_and_ids(metadata, uuids, match): if metadata["benchmark.keyword"] == "ingress-perf": return "ingress-performance", uuids index = "ripsaw-kube-burner" - runs = match.match_kube_burner(uuids) - return index, match.filter_runs(runs, runs) + if baseline == "": + runs = match.match_kube_burner(uuids) + ids = match.filter_runs(runs, runs) + else: + ids = uuids + return index, ids -def process_test(test, match, logger, output): +def process_test(test, match, output, uuid, baseline): """generate the dataframe for the test given Args: @@ -240,17 +254,25 @@ def process_test(test, match, logger, output): Returns: _type_: merged dataframe """ - metadata = get_metadata(test, logger) + logger= SingletonLogger(debug=logging.INFO).logger + if uuid in ('', None): + metadata = get_metadata(test) + else: + metadata = filter_metadata(uuid,match) logger.info("The test %s has started", test["name"]) uuids = match.get_uuid_by_metadata(metadata) - if len(uuids) == 0: - print("No UUID present for given metadata") - sys.exit() - - index, ids = get_index_and_ids(metadata, uuids, match) + if baseline in ('', None): + uuids = match.get_uuid_by_metadata(metadata) + if len(uuids) == 0: + logger.error("No UUID present for given metadata") + sys.exit() + else: + uuids = re.split(' |,',baseline) + uuids.append(uuid) + index, ids = get_index_and_ids(metadata, uuids, match, baseline) metrics = test["metrics"] - dataframe_list = get_metric_data(ids, index, metrics, match, logger) + dataframe_list = get_metric_data(ids, index, metrics, match) merged_df = reduce( lambda left, right: pd.merge(left, right, on="uuid", how="inner"), @@ -260,3 +282,47 @@ def process_test(test, match, logger, output): output_file_path = output.split(".")[0] + "-" + test["name"] + ".csv" match.save_results(merged_df, csv_file_path=output_file_path) return merged_df + +def filter_metadata(uuid,match): + """Gets metadata of the run from each test + + Args: + uuid (str): str of uuid ot find metadata of + match: the fmatch instance + + + Returns: + dict: dictionary of the metadata + """ + logger= SingletonLogger(debug=logging.INFO).logger + test = match.get_metadata_by_uuid(uuid) + metadata = { + 'platform': '', + 'clusterType': '', + 'masterNodesCount': 0, + 'workerNodesCount': 0, + 'infraNodesCount': 0, + 'masterNodesType': '', + 'workerNodesType': '', + 'infraNodesType': '', + 'totalNodesCount': 0, + 'ocpVersion': '', + 'networkType': '', + 'ipsec': '', + 'fips': '', + 'encrypted': '', + 'publish': '', + 'computeArch': '', + 'controlPlaneArch': '' + } + for k,v in test.items(): + if k not in metadata: + continue + metadata[k] = v + metadata['benchmark.keyword'] = test['benchmark'] + metadata["ocpVersion"] = str(metadata["ocpVersion"]) + + #Remove any keys that have blank values + no_blank_meta = {k: v for k, v in metadata.items() if v} + logger.debug('No blank metadata dict: ' + str(no_blank_meta)) + return no_blank_meta