diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..7da1f96 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 100 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9b4cda7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +settings.yaml +**/__pycache__/ +*.json +*.csv +.idea +trust diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..34d5395 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/asottile/reorder_python_imports + rev: v3.12.0 + hooks: + - id: reorder-python-imports + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-yaml + - id: debug-statements + - repo: https://github.com/psf/black + rev: 23.11.0 + hooks: + - id: black + - repo: https://github.com/pycqa/flake8 + rev: 6.1.0 + hooks: + - id: flake8 diff --git a/apix/diff.py b/apix/diff.py index 9aeb360..7e58211 100644 --- a/apix/diff.py +++ b/apix/diff.py @@ -1,10 +1,14 @@ # -*- encoding: utf-8 -*- """Determine the changes between two API versions.""" +from pathlib import Path + import attr import yaml -from pathlib import Path from logzero import logger -from apix.helpers import get_latest, get_previous, load_api + +from apix.helpers import get_latest +from apix.helpers import get_previous +from apix.helpers import load_api @attr.s() @@ -23,9 +27,7 @@ def __attrs_post_init__(self): self.api_name = get_latest(data_dir=self.data_dir, mock=self.mock) if not self.ver1: # get the latest saved version - self.ver1 = get_latest( - api_name=self.api_name, data_dir=self.data_dir, mock=self.mock - ) + self.ver1 = get_latest(api_name=self.api_name, data_dir=self.data_dir, mock=self.mock) if not self.ver2: # get the version before ver1 self.ver2 = get_previous(self.api_name, self.ver1, self.data_dir, self.mock) @@ -150,12 +152,13 @@ def save_diff(self, return_path=False): if self.mock: fpath = Path( - f"{self.data_dir}tests/APIs/{self.api_name}/{self.ver2}-to-{self.ver1}-diff.yaml" + f"{self.data_dir}tests/APIs/{self.api_name}" + f"/{self.ver2}-to-{self.ver1}-diff.yaml" ) else: ftype = "comp-diff" if self.compact else "diff" fpath = Path( - f"{self.data_dir}APIs/{self.api_name}/{self.ver2}-to-{self.ver1}-{ftype}.yaml" + f"{self.data_dir}APIs/{self.api_name}/" f"{self.ver2}-to-{self.ver1}-{ftype}.yaml" ) if fpath.exists(): fpath.unlink() diff --git a/apix/explore.py b/apix/explore.py index a3e1c1c..c0038b8 100644 --- a/apix/explore.py +++ b/apix/explore.py @@ -1,13 +1,16 @@ """Explore and API and save the results.""" -import aiohttp import asyncio +import time +from pathlib import Path + +import aiohttp import attr import requests -import time import yaml from logzero import logger -from pathlib import Path -from apix.parsers import apipie, test + +from apix.parsers import apipie +from apix.parsers import test @attr.s() @@ -35,7 +38,9 @@ def __attrs_post_init__(self): logger.warning("No known parser specified! Please review documentation.") async def _async_get(self, session, link): - """visit a page and download the content, returning the link and content""" + """ + visit a page and download the content returning the link and content + """ async with session.get(self.host_url + link[1], ssl=False) as response: content = await response.read() logger.debug(link[1]) @@ -53,15 +58,15 @@ async def _async_loop(self, links): self._queue.append(result) def _visit_links(self, links, retries=3): - """main controller for asynchronous page visiting, will attempt 3 retries""" + """ + Main controller for asynchronous page visiting, will attempt 3 retries + """ try: loop = asyncio.get_event_loop() loop.run_until_complete(self._async_loop(links)) - except aiohttp.client_exceptions.ServerDisconnectedError as err: + except aiohttp.client_exceptions.ServerDisconnectedError: logger.warning( - "Lost connection to host.{}".join( - "Retrying in 10 seconds" if retries else "" - ) + "Lost connection to host.".join("Retrying in 10 seconds" if retries else "") ) if retries: time.sleep(10) @@ -113,8 +118,7 @@ def explore(self): result = requests.get(self.host_url + self.base_path, verify=False) if not result: logger.warning( - f"I couldn't find anything useful at " - f"{self.host_url}{self.base_path}." + f"I couldn't find anything useful at " f"{self.host_url}{self.base_path}." ) return self.base_path = self.base_path.replace(".html", "") # for next step diff --git a/apix/helpers.py b/apix/helpers.py index 3a256a6..4a6d2a0 100644 --- a/apix/helpers.py +++ b/apix/helpers.py @@ -1,8 +1,9 @@ # -*- encoding: utf-8 -*- """A collection of miscellaneous helpers that don't quite fit in.""" -import yaml from copy import deepcopy from pathlib import Path + +import yaml from logzero import logger @@ -13,9 +14,7 @@ def get_api_list(data_dir=None, mock=False): if not api_dir.exists(): return None # get all versions in directory, that aren't diffs - apis = [ - (api.name, api.stat().st_mtime) for api in api_dir.iterdir() if api.is_dir() - ] or [] + apis = [(api.name, api.stat().st_mtime) for api in api_dir.iterdir() if api.is_dir()] or [] apis = [api for api, _ in sorted(apis, key=lambda x: x[1], reverse=True)] return apis @@ -33,9 +32,7 @@ def get_ver_list(api_name, data_dir=None, mock=False): versions = [ v_file.name.replace(".yaml", "") for v_file in save_path.iterdir() - if "-diff." not in v_file.name - and "-comp." not in v_file.name - and ".yaml" in v_file.name + if "-diff." not in v_file.name and "-comp." not in v_file.name and ".yaml" in v_file.name ] or [] return sorted(versions, reverse=True) @@ -86,12 +83,10 @@ def save_api(api_name, version, api_dict, data_dir=None, compact=False, mock=Fal """Save the dict to yaml, if the file doesn't exist""" if mock: a_path = Path( - f"{data_dir}tests/APIs/{api_name}/{version}{'-comp' if compact else ''}.yaml" + f"{data_dir}tests/APIs/{api_name}/{version}" f"{'-comp' if compact else ''}.yaml" ) else: - a_path = Path( - f"{data_dir}APIs/{api_name}/{version}{'-comp' if compact else ''}.yaml" - ) + a_path = Path(f"{data_dir}APIs/{api_name}/{version}" f"{'-comp' if compact else ''}.yaml") a_path.parent.mkdir(parents=True, exist_ok=True) logger.info(f"Saving {api_name} v{version} to {a_path}") with a_path.open("w") as f: diff --git a/apix/parsers/apipie.py b/apix/parsers/apipie.py index 8579c92..12a1f0d 100644 --- a/apix/parsers/apipie.py +++ b/apix/parsers/apipie.py @@ -22,13 +22,14 @@ class APIPie: def _compile_method(method_dict): """form the parameters and paths lists""" params = [ - f'{param["name"]} ~ {"required" if param["required"] else "optional"} ~ {param["expected_type"]}' + ( + f'{param["name"]} ~ ' + f'{"required" if param["required"] else "optional"} ~ ' + f'{param["expected_type"]}' + ) for param in method_dict["params"] ] - paths = [ - f'{path["http_method"].upper()} {path["api_url"]}' - for path in method_dict["apis"] - ] + paths = [f'{path["http_method"].upper()} {path["api_url"]}' for path in method_dict["apis"]] return {"paths": paths, "params": params} def scrape_content(self, result): @@ -38,12 +39,8 @@ def scrape_content(self, result): logger.debug(f"Compiling {name} with {len(data['methods'])} methods") self._data[name] = {"methods": []} for method in data["methods"]: - self._data[name]["methods"].append( - {method["name"]: self._compile_method(method)} - ) - self.params.update( - {param["name"]: param for param in method["params"]} - ) + self._data[name]["methods"].append({method["name"]: self._compile_method(method)}) + self.params.update({param["name"]: param for param in method["params"]}) def yaml_format(self, ingore=None): """Return the compiled data in a yaml-friendly format""" diff --git a/apix/parsers/test.py b/apix/parsers/test.py index b705b65..fe822b6 100644 --- a/apix/parsers/test.py +++ b/apix/parsers/test.py @@ -10,7 +10,6 @@ https://www.google.com/search?q=apix """ import attr -from logzero import logger from lxml import html @@ -47,12 +46,7 @@ def pull_links(result, base_path): links, last = [], None for link in g_links: url = link[2].replace("../", "") - if ( - "JacobCallahan" in url - and "sparkline" not in url - and link[0].text - and url != last - ): + if "JacobCallahan" in url and "sparkline" not in url and link[0].text and url != last: links.append((link[0].text, url)) last = url return links diff --git a/candore/__init__.py b/candore/__init__.py index e69de29..d8ac851 100644 --- a/candore/__init__.py +++ b/candore/__init__.py @@ -0,0 +1,60 @@ +import asyncio # noqa: F401 +import json +from pathlib import Path + +import click + +from candore.errors import ModeError +from candore.modules.api_lister import APILister +from candore.modules.comparator import Comparator +from candore.modules.extractor import Extractor +from candore.modules.report import Reporting + + +class Candore: + def __init__(self, settings): + self.settings = settings + self.api_lister = APILister(settings=self.settings) + + def list_endpoints(self): + return self.api_lister.lister_endpoints() + + async def save_all_entities(self, mode, output_file, full): + """Save all the entities to a json file + + :param mode: Pre or Post + :param output_file: Output file name + :param full: If True, save entities from all pages of the components, + else just saves first page + :return: None + """ + if mode not in ["pre", "post"]: + raise ModeError("Extracting mode must be 'pre' or 'post'") + + async with Extractor(settings=self.settings, apilister=self.api_lister) as extractor: + if full: + extractor.full = True + data = await extractor.extract_all_entities() + + if not data: + click.echo("Entities data is not data found!") + + file_path = Path(output_file) if output_file else Path(f"{mode}_entities.json") + with file_path.open(mode="w") as entfile: + json.dump(data, entfile) + click.echo(f"Entities data saved to {file_path}") + + def compare_entities( + self, + pre_file=None, + post_file=None, + output=None, + report_type=None, + record_evs=None, + ): + comp = Comparator(settings=self.settings) + if record_evs: + comp.record_evs = True + results = comp.compare_json(pre_file=pre_file, post_file=post_file) + reporter = Reporting(results=results) + reporter.generate_report(output_file=output, output_type=report_type) diff --git a/candore/candore.py b/candore/candore.py deleted file mode 100644 index 045d001..0000000 --- a/candore/candore.py +++ /dev/null @@ -1,50 +0,0 @@ -from candore.modules.api_lister import APILister -from candore.modules.extractor import Extractor -from candore.modules.comparator import Comparator -from candore.errors import ModeError -from candore.modules.report import Reporting -import click -import json -from pathlib import Path -import asyncio - - -api_lister = APILister() - - -def list_endpoints(): - return api_lister.lister_endpoints() - - -async def save_all_entities(mode, output_file, full): - """Save all the entities to a json file - - :param mode: Pre or Post - :param output_file: Output file name - :param full: If True, save entities from all pages of the components, else just saves first page - :return: None - """ - if mode not in ['pre', 'post']: - raise ModeError("Extracting mode must be 'pre' or 'post'") - - async with Extractor(apilister=api_lister) as extractor: - if full: - extractor.full = True - data = await extractor.extract_all_entities() - - if not data: - click.echo('Entities data is not data found!') - - file_path = Path(output_file) if output_file else Path(f'{mode}_entities.json') - with file_path.open(mode='w') as entfile: - json.dump(data, entfile) - click.echo(f'Entities data saved to {file_path}') - - -def compare_entities(pre_file=None, post_file=None, output=None, report_type=None, record_evs=None): - comp = Comparator() - if record_evs: - comp.record_evs = True - results = comp.compare_json(pre_file=pre_file, post_file=post_file) - reporter = Reporting(results=results) - reporter.generate_report(output_file=output, output_type=report_type) diff --git a/candore/cli.py b/candore/cli.py index a37d71e..8898a6a 100644 --- a/candore/cli.py +++ b/candore/cli.py @@ -1,9 +1,10 @@ -import click -from pprint import pprint -from candore.candore import list_endpoints -from candore.candore import save_all_entities -from candore.candore import compare_entities import asyncio +from pprint import pprint + +import click + +from candore import Candore +from candore.config import candore_settings # Click Interactive for Cloud Resources Cleanup @@ -13,13 +14,21 @@ invoke_without_command=True, ) @click.option("--version", is_flag=True, help="Installed version of candore") +@click.option("--settings-file", "-s", default=None, help="Settings file path") +@click.option("--components-file", "-c", default=None, help="Components file path") @click.pass_context -def candore(ctx, version): +def candore(ctx, version, settings_file, components_file): if version: import pkg_resources ver = pkg_resources.get_distribution("candore").version click.echo(f"Version: {ver}") + candore_obj = Candore( + settings=candore_settings( + option_settings_file=settings_file, option_components_file=components_file + ) + ) + ctx.__dict__["candore"] = candore_obj @candore.command(help="List API lister endpoints from Product") @@ -27,7 +36,8 @@ def candore(ctx, version): def apis(ctx): """List API lister endpoints from Product""" print("List of API lister endpoints from Product") - pprint(list_endpoints()) + candore_obj = ctx.parent.candore + pprint(candore_obj.list_endpoints()) @candore.command(help="Extract and save data using API lister endpoints") @@ -37,18 +47,32 @@ def apis(ctx): @click.pass_context def extract(ctx, mode, output, full): loop = asyncio.get_event_loop() - loop.run_until_complete(save_all_entities(mode=mode, output_file=output, full=full)) + candore_obj = ctx.parent.candore + loop.run_until_complete(candore_obj.save_all_entities(mode=mode, output_file=output, full=full)) @candore.command(help="Compare pre and post upgrade data") @click.option("--pre", type=str, help="The pre upgrade json file") @click.option("--post", type=str, help="The post upgrade json file") @click.option("-o", "--output", type=str, help="The output file name") -@click.option("-t", "--report-type", type=str, default='json', help="The type of report GSheet, JSON, or webpage") +@click.option( + "-t", + "--report-type", + type=str, + default="json", + help="The type of report GSheet, JSON, or webpage", +) @click.option("--record-evs", is_flag=True, help="Record Expected Variations in reporting") @click.pass_context def compare(ctx, pre, post, output, report_type, record_evs): - compare_entities(pre_file=pre, post_file=post, output=output, report_type=report_type, record_evs=record_evs) + candore_obj = ctx.parent.candore + candore_obj.compare_entities( + pre_file=pre, + post_file=post, + output=output, + report_type=report_type, + record_evs=record_evs, + ) if __name__ == "__main__": diff --git a/candore/config.py b/candore/config.py index c5245fc..df7c92d 100644 --- a/candore/config.py +++ b/candore/config.py @@ -1,30 +1,41 @@ -from pathlib import Path, PurePath +from pathlib import Path +from pathlib import PurePath + from dynaconf import Dynaconf from dynaconf.validator import Validator - CURRENT_DIRECTORY = Path().resolve() -settings_file = PurePath(CURRENT_DIRECTORY, 'settings.yaml') -components_file = PurePath(CURRENT_DIRECTORY, 'components.yaml') -# Initialize and Configure Settings -settings = Dynaconf( - core_loaders=["YAML"], - envvar_prefix="CANDORE", - settings_files=[settings_file, components_file], - envless_mode=True, - lowercase_read=True, -) -def validate_settings(): +def candore_settings(option_settings_file=None, option_components_file=None): + settings_file = ( + PurePath(option_settings_file) + if option_settings_file + else PurePath(CURRENT_DIRECTORY, "settings.yaml") + ) + components_file = ( + PurePath(option_components_file) + if option_components_file + else PurePath(CURRENT_DIRECTORY, "components.yaml") + ) + # Initialize and Configure Settings + settings = Dynaconf( + core_loaders=["YAML"], + envvar_prefix="CANDORE", + settings_files=[settings_file, components_file], + envless_mode=True, + lowercase_read=True, + ) + validate_settings(settings) + return settings + + +def validate_settings(settings): provider_settings = [ - f"candore.{setting_key}" for setting_key in settings.to_dict().get('CANDORE') + f"candore.{setting_key}" for setting_key in settings.to_dict().get("CANDORE") ] settings.validators.register(Validator(*provider_settings, ne=None)) try: settings.validators.validate() except Exception as ecc: raise ecc - - -validate_settings() diff --git a/candore/errors.py b/candore/errors.py index a726d98..3912c5b 100644 --- a/candore/errors.py +++ b/candore/errors.py @@ -1,8 +1,9 @@ # List of custom exceptions used by Candore modules + class NoDataFound(Exception): pass + class ModeError(Exception): pass - diff --git a/candore/modules/api_lister.py b/candore/modules/api_lister.py index daac7b7..7e53695 100644 --- a/candore/modules/api_lister.py +++ b/candore/modules/api_lister.py @@ -1,13 +1,15 @@ from apix.explore import AsyncExplorer -from candore.config import settings -from candore.config import validate_settings - class APILister: - - def __init__(self): - self.explorer = AsyncExplorer(name=settings.candore.product_name, version=settings.candore.version, host_url=settings.candore.base_url, base_path=settings.candore.docpath, parser=settings.candore.parser) + def __init__(self, settings): + self.explorer = AsyncExplorer( + name=settings.candore.product_name, + version=settings.candore.version, + host_url=settings.candore.base_url, + base_path=settings.candore.docpath, + parser=settings.candore.parser, + ) self.list_endpoints = {} def _endpoints(self): @@ -19,16 +21,14 @@ def lister_endpoints(self): list_endpoints = {} for component, data in self._endpoints().items(): - methods = data.get('methods', []) + methods = data.get("methods", []) component_list_apis = [ - path.lstrip('GET ') for method in methods - for path in method.get('index', {}).get('paths', []) - if 'id' not in path + path.lstrip("GET ") + for method in methods + for path in method.get("index", {}).get("paths", []) + if "id" not in path ] list_endpoints[component] = component_list_apis self.list_endpoints = list_endpoints return list_endpoints - - -apilister = APILister() diff --git a/candore/modules/comparator.py b/candore/modules/comparator.py index 22becd7..ebafb22 100644 --- a/candore/modules/comparator.py +++ b/candore/modules/comparator.py @@ -1,21 +1,22 @@ import json -from candore.utils import last_index_of_element + from candore.modules.variatons import Variations +from candore.utils import last_index_of_element class Comparator: - - def __init__(self): + def __init__(self, settings): self.big_key = [] self.big_compare = {} self.record_evs = False - self.variations = Variations() + self.variations = Variations(settings) self.expected_variations = self.variations.expected_variations self.skipped_variations = self.variations.skipped_variations def remove_non_variant_key(self, key): reversed_bk = self.big_key[::-1] - reversed_bk.remove(key) + if key in reversed_bk: + reversed_bk.remove(key) self.big_key = reversed_bk[::-1] def remove_path(self, identy): @@ -26,14 +27,21 @@ def remove_path(self, identy): def record_variation(self, pre, post, var_details=None): big_key = [str(itm) for itm in self.big_key] - full_path = '/'.join(big_key) - var_full_path = '/'.join([itm for itm in self.big_key if not isinstance(itm, int)]) + full_path = "/".join(big_key) + var_full_path = "/".join([itm for itm in self.big_key if not isinstance(itm, int)]) if var_full_path in self.expected_variations or var_full_path in self.skipped_variations: if self.record_evs: - variation = {'pre': pre, 'post': post, 'variation': var_details or 'Expected(A)'} + variation = { + "pre": pre, + "post": post, + "variation": var_details or "Expected(A)", + } self.big_compare.update({full_path: variation}) - elif var_full_path not in self.expected_variations and var_full_path not in self.skipped_variations: - variation = {'pre': pre, 'post': post, 'variation': var_details or ''} + elif ( + var_full_path not in self.expected_variations + and var_full_path not in self.skipped_variations + ): + variation = {"pre": pre, "post": post, "variation": var_details or ""} self.big_compare.update({full_path: variation}) def _is_data_type_dict(self, pre, post): @@ -43,9 +51,13 @@ def _is_data_type_dict(self, pre, post): self.compare_all_pres_with_posts(pre[key], post[key], unique_key=key) else: self.compare_all_pres_with_posts( - pre[pre_key], None, unique_key=pre_key, var_details='Post lookup key missing') + pre[pre_key], + None, + unique_key=pre_key, + var_details="Post lookup key missing", + ) - def _is_data_type_list(self, pre, post, unique_key=''): + def _is_data_type_list(self, pre, post, unique_key=""): for pre_entity in pre: if not pre_entity: continue @@ -53,15 +65,19 @@ def _is_data_type_list(self, pre, post, unique_key=''): for post_entity in post: if not post_entity: continue - if 'id' in pre_entity: - if pre_entity['id'] == post_entity['id']: - self.compare_all_pres_with_posts(pre_entity, post_entity, unique_key=pre_entity['id']) + if "id" in pre_entity: + if pre_entity["id"] == post_entity["id"]: + self.compare_all_pres_with_posts( + pre_entity, post_entity, unique_key=pre_entity["id"] + ) else: key = list(pre_entity.keys())[0] if pre_entity[key] == post_entity[key]: - self.compare_all_pres_with_posts(pre_entity[key], post_entity[key], unique_key=key) - if 'id' in pre_entity: - self.remove_path(pre_entity['id']) + self.compare_all_pres_with_posts( + pre_entity[key], post_entity[key], unique_key=key + ) + if "id" in pre_entity: + self.remove_path(pre_entity["id"]) else: self.remove_path(pre_entity[list(pre_entity.keys())[0]]) else: @@ -69,7 +85,7 @@ def _is_data_type_list(self, pre, post, unique_key=''): self.record_variation(pre, post) self.remove_path(unique_key) - def compare_all_pres_with_posts(self, pre_data, post_data, unique_key='', var_details=None): + def compare_all_pres_with_posts(self, pre_data, post_data, unique_key="", var_details=None): if unique_key: self.big_key.append(unique_key) if type(pre_data) is dict: @@ -82,7 +98,7 @@ def compare_all_pres_with_posts(self, pre_data, post_data, unique_key='', var_de self.remove_non_variant_key(unique_key) def compare_json(self, pre_file, post_file): - pre_data = post_Data = None + pre_data = post_data = None with open(pre_file, "r") as fpre: pre_data = json.load(fpre) diff --git a/candore/modules/extractor.py b/candore/modules/extractor.py index 34aa800..b7d1a48 100644 --- a/candore/modules/extractor.py +++ b/candore/modules/extractor.py @@ -1,18 +1,19 @@ +import asyncio # noqa: F401 +from functools import cached_property + import aiohttp -from candore.config import settings -from functools import cached_property -import asyncio class Extractor: - def __init__(self, apilister=None): + def __init__(self, settings, apilister=None): """Extract and save data using API lister endpoints :param apilister: APILister object """ - self.username = settings.candore.username - self._passwd = settings.candore.password - self.base = settings.candore.base_url + self.settings = settings + self.username = self.settings.candore.username + self._passwd = self.settings.candore.password + self.base = self.settings.candore.base_url self.verify_ssl = False self.auth = aiohttp.BasicAuth(self.username, self._passwd) self.connector = aiohttp.TCPConnector(ssl=self.verify_ssl) @@ -22,13 +23,13 @@ def __init__(self, apilister=None): @cached_property def dependent_components(self): - if hasattr(settings, 'components'): - return settings.components.dependencies + if hasattr(self.settings, "components"): + return self.settings.components.dependencies @cached_property def ignore_components(self): - if hasattr(settings, 'components'): - return settings.components.ignore + if hasattr(self.settings, "components"): + return self.settings.components.ignore @cached_property def api_endpoints(self): @@ -53,34 +54,38 @@ async def paged_results(self, **get_params): async with self.client.get(**get_params) as response: if response.status == 200: _paged_results = await response.json() - _paged_results = _paged_results.get('results') + _paged_results = _paged_results.get("results") return _paged_results async def fetch_component_entities(self, **comp_params): entity_data = [] - endpoint = comp_params.get('endpoint', None) - data = comp_params.get('data') - dependency = comp_params.get('dependency', None) - _request = {'url': self.base+'/'+endpoint, 'params': {}} + endpoint = comp_params.get("endpoint", None) + data = comp_params.get("data") + dependency = comp_params.get("dependency", None) + _request = {"url": self.base + "/" + endpoint, "params": {}} if data and dependency: - _request['params'].update({f'{dependency}_id': data}) + _request["params"].update({f"{dependency}_id": data}) async with self.client.get(**_request) as response: if response.status == 200: results = await response.json() - if 'results' in results: - entities = results.get('results') + if "results" in results: + entities = results.get("results") entity_data.extend(entities) else: - # Return an empty directory for endpoints like services, api etc + # Return an empty directory for endpoints + # like services, api etc # which does not have results return entity_data # If the entity has multiple pages, fetch them all if self.full: - total_pages = results.get('total') // results.get('per_page') + 1 + total_pages = results.get("total") // results.get("per_page") + 1 if total_pages > 1: - print(f'Endpoint {endpoint} has {total_pages} pages. This would take some time ....') - for page in range(2, total_pages+1): - _request['params'].update({'page': page}) + print( + f"Endpoint {endpoint} has {total_pages} pages. " + "This would take some time ...." + ) + for page in range(2, total_pages + 1): + _request["params"].update({"page": page}) page_entities = await self.paged_results(**_request) entity_data.extend(page_entities) return entity_data @@ -88,12 +93,11 @@ async def fetch_component_entities(self, **comp_params): async def dependency_ids(self, dependency): # All the Ids of a specific dependency # e.g Organization IDs 1, 2, 3, 4 - endpoint = self.api_endpoints[f'{dependency}s'][0] + endpoint = self.api_endpoints[f"{dependency}s"][0] depe_lists = await self.fetch_component_entities(endpoint=endpoint) - depen_ids = [dep_dict['id'] for dep_dict in depe_lists] + depen_ids = [dep_dict["id"] for dep_dict in depe_lists] return depen_ids - async def component_params(self, component_endpoint): """ component_endpoints = ['katello/api/activationkeys'] @@ -104,7 +108,7 @@ async def component_params(self, component_endpoint): data = {} dependency = None # remove ignored endpoints - _last = component_endpoint.rsplit('/')[-1] + _last = component_endpoint.rsplit("/")[-1] # Ignorable endpoint if self.ignore_components and _last in self.ignore_components: return @@ -112,8 +116,7 @@ async def component_params(self, component_endpoint): if self.dependent_components and _last in self.dependent_components.keys(): dependency = self.dependent_components[_last] data = await self.dependency_ids(dependency) - return {'endpoint': component_endpoint, 'data': data, 'dependency': dependency} - + return {"endpoint": component_endpoint, "data": data, "dependency": dependency} async def process_entities(self, endpoints): """ @@ -125,10 +128,13 @@ async def process_entities(self, endpoints): comp_params = await self.component_params(component_endpoint=endpoint) if comp_params: entities = [] - if isinstance(comp_params.get('data'), list): - for data_point in comp_params.get('data'): + if isinstance(comp_params.get("data"), list): + for data_point in comp_params.get("data"): depen_data = await self.fetch_component_entities( - endpoint=comp_params['endpoint'], dependency=comp_params.get('dependency'), data=data_point) + endpoint=comp_params["endpoint"], + dependency=comp_params.get("dependency"), + data=data_point, + ) if not depen_data: continue entities.extend(depen_data) @@ -139,8 +145,8 @@ async def process_entities(self, endpoints): return comp_data async def extract_all_entities(self): - """ - component, endpoints = `activation_key`, ['katello/api/activationkeys'] + """Extract all entities fom all endpoints + :return: """ all_data = {} diff --git a/candore/modules/report.py b/candore/modules/report.py index 953a23a..00c3d75 100644 --- a/candore/modules/report.py +++ b/candore/modules/report.py @@ -1,7 +1,8 @@ """Centralized Reporting utilities for compared results""" -import json import csv +import json from pathlib import Path + # from .webapp import display_json_table, render_webpage @@ -21,17 +22,17 @@ def generate_report(self, output_file, output_type): Args: output_file (str): The file to write the report to - output_type (str): The type of report to generate (json, CSV or Web) + output_type (str): The type of report to generate json / CSV Returns: None Raises: ValueError: If the output_type is not supported """ - if output_type == 'json': + if output_type == "json": self._generate_json_report(output_file) - elif output_type == 'html': + elif output_type == "html": self._generate_html_report() - elif output_type == 'csv': + elif output_type == "csv": self._generate_csv_report(output_file) else: raise ValueError("Output type {} not supported".format(output_type)) @@ -45,7 +46,7 @@ def _generate_json_report(self, output_file): None """ if not output_file: - output_file = 'results.json' + output_file = "results.json" output_file = Path(output_file) # Write the JSON report to the output file output_file.write_text(json.dumps(self.results, indent=4)) @@ -60,8 +61,8 @@ def _generate_html_report(self): Returns: None """ - #display_json_table(results_json=self.results) - #render_webpage() + # display_json_table(results_json=self.results) + # render_webpage() print("HTML report is ready to view at: http://localhost:5000") def _generate_csv_report(self, output_file): @@ -73,13 +74,12 @@ def _generate_csv_report(self, output_file): None """ if not output_file: - output_file = 'results.csv' + output_file = "results.csv" output_file = Path(output_file) # Convert json to csv and write to output file - csv_writer = csv.writer(output_file.open('w')) - csv_writer.writerow(['Variation Path', 'Pre-Upgrade', 'Post-Upgrade', 'Variation']) + csv_writer = csv.writer(output_file.open("w")) + csv_writer.writerow(["Variation Path", "Pre-Upgrade", "Post-Upgrade", "Variation"]) for var_path, vals in self.results.items(): - csv_writer.writerow([var_path, vals['pre'], vals['post'], vals['variation']]) + csv_writer.writerow([var_path, vals["pre"], vals["post"], vals["variation"]]) print("Wrote CSV report to {}".format(output_file)) print("CSV report contains {} results".format(len(self.results))) - diff --git a/candore/modules/variatons.py b/candore/modules/variatons.py index cda17d3..d42d312 100644 --- a/candore/modules/variatons.py +++ b/candore/modules/variatons.py @@ -2,14 +2,17 @@ A module responsible for calculating expected and skipped variations from `conf/variations` yaml file and convert them into processable list """ -import yaml -from pathlib import Path -from candore.config import settings, CURRENT_DIRECTORY from functools import cached_property +from pathlib import Path + +import yaml class Variations: - def get_paths(self, variations, prefix='', separator='/'): + def __init__(self, settings): + self.settings = settings + + def get_paths(self, variations, prefix="", separator="/"): paths = [] if isinstance(variations, dict): for key, value in variations.items(): @@ -18,13 +21,13 @@ def get_paths(self, variations, prefix='', separator='/'): for item in variations: paths.extend(self.get_paths(item, prefix, separator)) else: - paths.append(f'{prefix}{variations}') + paths.append(f"{prefix}{variations}") return paths @cached_property def variations(self): - templates_path = Path(CURRENT_DIRECTORY, settings.candore.var_file) + templates_path = Path(self.settings.candore.var_file) if not templates_path.exists(): print(f"The file {templates_path} does not exist.") with templates_path.open() as yaml_file: @@ -33,8 +36,8 @@ def variations(self): @cached_property def expected_variations(self): - return self.get_paths(variations=self.variations.get('expected_variations')) + return self.get_paths(variations=self.variations.get("expected_variations")) @cached_property def skipped_variations(self): - return self.get_paths(variations=self.variations.get('skipped_variations')) + return self.get_paths(variations=self.variations.get("skipped_variations")) diff --git a/candore/modules/webapp.py b/candore/modules/webapp.py index 6f7144d..4088643 100644 --- a/candore/modules/webapp.py +++ b/candore/modules/webapp.py @@ -1,14 +1,15 @@ -from flask import Flask, render_template +from flask import Flask +from flask import render_template app = Flask(__name__) # Define a route to display the JSON data in a table -@app.route('/') +@app.route("/") def display_json_table(results_json=None): # Fetch JSON data from your module (replace with your data source) - return render_template('table.html', data=results_json) + return render_template("table.html", data=results_json) def render_webpage(): diff --git a/pyproject.toml b/pyproject.toml index 06e3ae8..9fc0f43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,9 +49,9 @@ dependencies = [ ] [project.optional-dependencies] -satellite = ["satellite.can"] test = [ "mock", + "pre-commit", "pytest", "pytest-cov", "pytest-mock", @@ -60,4 +60,21 @@ test = [ ] [project.scripts] -candore = "candore.cli:candore" \ No newline at end of file +candore = "candore.cli:candore" + +[tool.black] +line-length = 100 +skip-string-normalization = true +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.hg + | \.mypy_cache + | \.venv + | _build + | buck-out + | build + | dist +)/ +''' diff --git a/tests/test_listers.py b/tests/test_listers.py index 99188b1..615b388 100644 --- a/tests/test_listers.py +++ b/tests/test_listers.py @@ -2,40 +2,39 @@ class TestLister: - instance = APILister() def test_list_endpoints(self): # Mock the _apis() method to return sample data self.instance._endpoints = lambda: { - 'AK': { - 'methods': [ - {'index': {'paths': ['GET /api/aks', 'GET /api/aks/cvs']}}, - {'create': {'paths': ['POST /api/aks']}} + "AK": { + "methods": [ + {"index": {"paths": ["GET /api/aks", "GET /api/aks/cvs"]}}, + {"create": {"paths": ["POST /api/aks"]}}, ] }, - 'CV': { - 'methods': [ - {'index': {'paths': ['GET /api/cvs/']}}, - {'delete': {'paths': ['DELETE /api/cvs']}} + "CV": { + "methods": [ + {"index": {"paths": ["GET /api/cvs/"]}}, + {"delete": {"paths": ["DELETE /api/cvs"]}}, ] }, - 'PRODS': { - 'methods': [ - {'index': {'paths': ['GET /api/prod_id/repos/']}}, - {'delete': {'paths': ['DELETE /api/prods']}} + "PRODS": { + "methods": [ + {"index": {"paths": ["GET /api/prod_id/repos/"]}}, + {"delete": {"paths": ["DELETE /api/prods"]}}, ] - } + }, } expected_result = { - 'AK': ['/api/aks', '/api/aks/cvs'], - 'CV': ['/api/cvs/'], - 'PRODS': [] + "AK": ["/api/aks", "/api/aks/cvs"], + "CV": ["/api/cvs/"], + "PRODS": [], } # Call the list_apis method result = self.instance.lister_endpoints() # Assert that the result matches the expected output - assert result == expected_result, 'The API endpoints list differs' + assert result == expected_result, "The API endpoints list differs"