From a22513b65255d2342eb095197b1f911166594df6 Mon Sep 17 00:00:00 2001 From: jyejare Date: Fri, 19 Apr 2024 18:00:21 +0530 Subject: [PATCH] Adding expected constants support and better var file examples --- 6_14.yaml | 17 -------------- 6_14_constants.yaml.template | 10 ++++++++ 6_14_variations.yaml.template | 10 ++++++++ candore/modules/comparator.py | 37 +++++++++++++++++++++-------- candore/modules/report.py | 4 +++- candore/modules/variations.py | 44 +++++++++++++++++++++++++++++++++++ candore/modules/variatons.py | 43 ---------------------------------- candore/utils.py | 24 +++++++++++++++++++ settings.yaml.template | 3 ++- 9 files changed, 120 insertions(+), 72 deletions(-) delete mode 100644 6_14.yaml create mode 100644 6_14_constants.yaml.template create mode 100644 6_14_variations.yaml.template create mode 100644 candore/modules/variations.py delete mode 100644 candore/modules/variatons.py diff --git a/6_14.yaml b/6_14.yaml deleted file mode 100644 index d47b4f5..0000000 --- a/6_14.yaml +++ /dev/null @@ -1,17 +0,0 @@ -expected_variations: [] -# hosts: -# - realm_id -# - capabilities -# smart_proxies: -# features: name - -skipped_variations: - hosts: - puppet_proxy: - content_facet_attributes: - errata_counts: - applicable: - content_view: - lifecycle_environment: - subscription_facet_attributes: - reported_data: uptime_seconds diff --git a/6_14_constants.yaml.template b/6_14_constants.yaml.template new file mode 100644 index 0000000..1b618f3 --- /dev/null +++ b/6_14_constants.yaml.template @@ -0,0 +1,10 @@ +expected_constants: + # These are just examples, please replace these with some real expected constants + hosts: + content_facet_something: content_source_something_id + +skipped_constants: + # These are just examples, please replace these with some real skip-able constants + content_view_versions: + environments: + permissions_something: something here diff --git a/6_14_variations.yaml.template b/6_14_variations.yaml.template new file mode 100644 index 0000000..2450641 --- /dev/null +++ b/6_14_variations.yaml.template @@ -0,0 +1,10 @@ +expected_variations: + # These are just examples, please replace these with some real expected variations + hosts: + content_facet_somevar: content_source_ssomevar_id + +skipped_variations: + # These are just examples, please replace these with some real skip-able variations + content_view_versions: + environments: + permissions_somevar: somevar_here diff --git a/candore/modules/comparator.py b/candore/modules/comparator.py index 19bab14..bc9ba67 100644 --- a/candore/modules/comparator.py +++ b/candore/modules/comparator.py @@ -1,18 +1,23 @@ import json -from candore.modules.variatons import Variations +from candore.modules.variations import Variations, Constants from candore.utils import last_index_of_element, is_list_contains_dict class Comparator: def __init__(self, settings): self.big_key = [] - self.big_compare = {} + self.big_diff = {} self.big_constant = {} self.record_evs = False self.variations = Variations(settings) + self.constants = Constants(settings) self.expected_variations = self.variations.expected_variations self.skipped_variations = self.variations.skipped_variations + self.expected_constants = self.constants.expected_constants + self.skipped_constants = self.constants.skipped_constants + + def remove_verifed_key(self, key): reversed_bk = self.big_key[::-1] @@ -37,20 +42,32 @@ def record_variation(self, pre, post, var_details=None): "post": post, "variation": var_details or "Expected(A)", } - self.big_compare.update({full_path: variation}) + self.big_diff.update({full_path: variation}) elif ( var_full_path not in self.expected_variations and var_full_path not in self.skipped_variations ): variation = {"pre": pre, "post": post, "variation": var_details or ""} - self.big_compare.update({full_path: variation}) + self.big_diff.update({full_path: variation}) - def record_constants(self, pre, post): + def record_constants(self, pre, post, var_details=None): big_key = [str(itm) for itm in self.big_key] full_path = "/".join(big_key) - # var_full_path = "/".join([itm for itm in self.big_key if not isinstance(itm, int)]) - variation = {"pre": pre, "post": post} - self.big_constant.update({full_path: variation}) + var_full_path = "/".join([itm for itm in self.big_key if not isinstance(itm, int)]) + if var_full_path in self.expected_constants or var_full_path in self.skipped_constants: + if self.record_evs: + variation = { + "pre": pre, + "post": post, + "constant": var_details or "Expected(A)", + } + self.big_constant.update({full_path: variation}) + elif ( + var_full_path not in self.expected_constants + and var_full_path not in self.skipped_constants + ): + variation = {"pre": pre, "post": post, "constant": var_details or ""} + self.big_constant.update({full_path: variation}) def _is_data_type_dict(self, pre, post, unique_key=""): if (pre and 'id' in pre) and (post and 'id' in post): @@ -118,7 +135,7 @@ def compare_all_pres_with_posts(self, pre_data, post_data, unique_key="", var_de if pre_data != post_data: self.record_variation(pre_data, post_data, var_details) else: - self.record_constants(pre_data, post_data) + self.record_constants(pre_data, post_data, var_details) self.remove_verifed_key(unique_key) def compare_json(self, pre_file, post_file, inverse): @@ -132,6 +149,6 @@ def compare_json(self, pre_file, post_file, inverse): self.compare_all_pres_with_posts(pre_data, post_data) if not inverse: - return self.big_compare + return self.big_diff else: return self.big_constant diff --git a/candore/modules/report.py b/candore/modules/report.py index ea14725..40400b0 100644 --- a/candore/modules/report.py +++ b/candore/modules/report.py @@ -84,6 +84,8 @@ def _generate_csv_report(self, output_file, inverse): csv_writer.writerow(columns) # Writing Rows for var_path, vals in self.results.items(): - csv_writer.writerow([var_path, vals["pre"], vals["post"], vals.get('variation', None)]) + csv_writer.writerow([ + var_path, vals["pre"], vals["post"], + vals["variation" if not inverse else "constant"]]) print("Wrote CSV report to {}".format(output_file)) print("CSV report contains {} results".format(len(self.results))) diff --git a/candore/modules/variations.py b/candore/modules/variations.py new file mode 100644 index 0000000..735bb2a --- /dev/null +++ b/candore/modules/variations.py @@ -0,0 +1,44 @@ +""" +A module responsible for calculating expected and skipped variations from +`conf/variations` yaml file and convert them into processable list +""" +from functools import cached_property +from candore.utils import yaml_reader, get_yaml_paths + +import yaml + + +class Variations: + def __init__(self, settings): + self.settings = settings + + @cached_property + def variations(self): + yaml_data = yaml_reader(file_path=self.settings.candore.var_file) + return yaml_data + + @cached_property + def expected_variations(self): + return get_yaml_paths(yaml_data=self.variations.get("expected_variations")) + + @cached_property + def skipped_variations(self): + return get_yaml_paths(yaml_data=self.variations.get("skipped_variations")) + + +class Constants: + def __init__(self, settings): + self.settings = settings + + @cached_property + def constants(self): + yaml_data = yaml_reader(file_path=self.settings.candore.constant_file) + return yaml_data + + @cached_property + def expected_constants(self): + return get_yaml_paths(yaml_data=self.constants.get("expected_constants")) + + @cached_property + def skipped_constants(self): + return get_yaml_paths(yaml_data=self.constants.get("skipped_constants")) diff --git a/candore/modules/variatons.py b/candore/modules/variatons.py deleted file mode 100644 index d42d312..0000000 --- a/candore/modules/variatons.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -A module responsible for calculating expected and skipped variations from -`conf/variations` yaml file and convert them into processable list -""" -from functools import cached_property -from pathlib import Path - -import yaml - - -class Variations: - def __init__(self, settings): - self.settings = settings - - def get_paths(self, variations, prefix="", separator="/"): - paths = [] - if isinstance(variations, dict): - for key, value in variations.items(): - paths.extend(self.get_paths(value, f"{prefix}{key}{separator}")) - elif isinstance(variations, list): - for item in variations: - paths.extend(self.get_paths(item, prefix, separator)) - else: - paths.append(f"{prefix}{variations}") - - return paths - - @cached_property - def variations(self): - templates_path = Path(self.settings.candore.var_file) - if not templates_path.exists(): - print(f"The file {templates_path} does not exist.") - with templates_path.open() as yaml_file: - yaml_data = yaml.safe_load(yaml_file) - return yaml_data - - @cached_property - def expected_variations(self): - return self.get_paths(variations=self.variations.get("expected_variations")) - - @cached_property - def skipped_variations(self): - return self.get_paths(variations=self.variations.get("skipped_variations")) diff --git a/candore/utils.py b/candore/utils.py index 9b759e4..d041b1a 100644 --- a/candore/utils.py +++ b/candore/utils.py @@ -1,6 +1,8 @@ """ An utility helpers module """ +from pathlib import Path +import yaml def last_index_of_element(arr, element): @@ -13,3 +15,25 @@ def last_index_of_element(arr, element): def is_list_contains_dict(_list): contains_dict = any(isinstance(element, dict) for element in _list) return contains_dict + + +def yaml_reader(file_path=None): + templates_path = Path(file_path) + if not templates_path.exists(): + print(f"The file {templates_path} does not exist.") + with templates_path.open() as yaml_file: + yaml_data = yaml.safe_load(yaml_file) + return yaml_data + + +def get_yaml_paths(yaml_data, prefix="", separator="/"): + paths = [] + if isinstance(yaml_data, dict): + for key, value in yaml_data.items(): + paths.extend(get_yaml_paths(value, f"{prefix}{key}{separator}")) + elif isinstance(yaml_data, list): + for item in yaml_data: + paths.extend(get_yaml_paths(item, prefix, separator)) + else: + paths.append(f"{prefix}{yaml_data}") + return paths \ No newline at end of file diff --git a/settings.yaml.template b/settings.yaml.template index e56b247..7cd690d 100644 --- a/settings.yaml.template +++ b/settings.yaml.template @@ -6,4 +6,5 @@ candore: product_version: 6.14 docpath: /apidoc parser: apipie - var_file: "@jinja conf/variations/{{this.candore.version | replace('.', '_')}}.yaml" + var_file: "@jinja {{this.candore.product_version | replace('.', '_')}}_variations.yaml" + constant_File: "@jinja {{this.candore.product_version | replace('.', '_')}}_constants.yaml"