diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2a8d60a..4813d29 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -5,6 +5,10 @@ on: branches: [master] pull_request: types: [opened, synchronize, reopened] + paths-ignore: + - "*.md" + - "*.example" + - ".gitignore" jobs: analyze: @@ -14,22 +18,24 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.10", "3.11", "3.12"] steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + ref: ${{ github.ref }} - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -38,5 +44,4 @@ jobs: pip install -U pip pip install -U .[test] cp manifester_settings.yaml.example manifester_settings.yaml - manifester --help - # pytest -v tests/ --ignore tests/functional + pytest -v tests/ diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 73b8408..5a261ba 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -12,12 +12,12 @@ jobs: strategy: matrix: # build/push in lowest support python version - python-version: [ 3.8 ] + python-version: [ 3.10 ] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -29,7 +29,7 @@ jobs: python -m twine check dist/* - name: Build and publish - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@v1.8.11 with: user: __token__ password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/update_manifester_image.yml b/.github/workflows/update_manifester_image.yml index a92d385..3f43f64 100644 --- a/.github/workflows/update_manifester_image.yml +++ b/.github/workflows/update_manifester_image.yml @@ -11,16 +11,16 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to Quay Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ${{ secrets.QUAY_SERVER }} username: ${{ secrets.QUAY_USERNAME }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c482f05..7980a70 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,27 +1,19 @@ # configuration for pre-commit git hooks repos: -- repo: https://github.com/asottile/reorder_python_imports - rev: v3.0.1 - hooks: - - id: reorder-python-imports -- repo: https://github.com/asottile/pyupgrade - rev: v2.32.0 - hooks: - - id: pyupgrade - args: [--py36-plus] -- repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 - hooks: - - id: flake8 -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - - id: debug-statements + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: check-yaml + - id: debug-statements + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.6 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format + - repo: https://github.com/gitleaks/gitleaks + rev: v8.18.0 + hooks: + - id: gitleaks diff --git a/manifester/commands.py b/manifester/commands.py index 258c561..00e6bbd 100644 --- a/manifester/commands.py +++ b/manifester/commands.py @@ -1,3 +1,4 @@ +"""Defines the CLI commands for Manifester.""" import click from manifester import Manifester @@ -6,6 +7,7 @@ # To do: add a command for returning subscription pools @click.group def cli(): + """Command-line interface for manifester.""" pass @@ -15,10 +17,9 @@ def cli(): type=str, help="Category of manifest (golden_ticket or robottelo_automation by default)", ) -@click.option( - "--allocation_name", type=str, help="Name of upstream subscription allocation" -) +@click.option("--allocation_name", type=str, help="Name of upstream subscription allocation") def get_manifest(manifest_category, allocation_name): + """Return a subscription manifester based on the settings for the provided manifest_category.""" manifester = Manifester(manifest_category, allocation_name) manifester.create_subscription_allocation() for sub in manifester.subscription_data: diff --git a/manifester/helpers.py b/manifester/helpers.py index 08e6a26..3be059c 100644 --- a/manifester/helpers.py +++ b/manifester/helpers.py @@ -1,10 +1,13 @@ +"""Defines helper functions used by Manifester.""" +from collections import UserDict +import random import time from logzero import logger def simple_retry(cmd, cmd_args=None, cmd_kwargs=None, max_timeout=240, _cur_timeout=1): - """Re(Try) a function given its args and kwargs up until a max timeout""" + """Re(Try) a function given its args and kwargs up until a max timeout.""" cmd_args = cmd_args if cmd_args else [] cmd_kwargs = cmd_kwargs if cmd_kwargs else {} # If additional debug information is needed, the following log entry can be modified to @@ -16,26 +19,81 @@ def simple_retry(cmd, cmd_args=None, cmd_kwargs=None, max_timeout=240, _cur_time if response.status_code in [429, 500, 504]: new_wait = _cur_timeout * 2 if new_wait > max_timeout: - raise Exception("Timeout exceeded") + raise Exception("Retry timeout exceeded") logger.debug(f"Trying again in {_cur_timeout} seconds") time.sleep(_cur_timeout) response = simple_retry(cmd, cmd_args, cmd_kwargs, max_timeout, new_wait) return response + def process_sat_version(sat_version, valid_sat_versions): - """Ensure that the sat_version parameter is properly formatted for the RHSM API when creating - a subscription allocation with the 'POST allocations' endpoint""" + """Ensure that the sat_version parameter is properly formatted for the RHSM API.""" + expected_length = 8 if sat_version not in valid_sat_versions: # The valid values for the sat_version parameter when creating a subscription allocation # are all 8 characters or less (e.g. 'sat-6.11'). Some data sources may include a Z-stream # version (e.g. 'sat-6.11.0') when retrieving this value from settings. The conditional # below assumes that, if the length of sat_version is greated than 8 characters, it includes # a Z-stream version that should be removed. - if len(sat_version) > 8: - sat_version = sat_version.split('.') + if len(sat_version) > expected_length: + sat_version = sat_version.split(".") sat_version = sat_version[0:2] sat_version = ".".join(sat_version) # If sat_version is still not valid, default to the latest valid version. if sat_version not in valid_sat_versions: - valid_sat_versions.sort(key = lambda i: int(i.split('-')[-1].split('.')[-1]), reverse = True) - return valid_sat_versions[0] + valid_sat_versions.sort( + key=lambda i: int(i.split("-")[-1].split(".")[-1]), reverse=True + ) + return valid_sat_versions[0] + return sat_version + + +def fake_http_response_code(good_codes=None, bad_codes=None, fail_rate=0): + """Return an HTTP response code randomly selected from sets of good and bad codes.""" + if random.random() > (fail_rate / 100): + return random.choice(good_codes) + else: + return random.choice(bad_codes) + + +class MockStub(UserDict): + """Test helper class. Allows for both arbitrary mocking and stubbing.""" + + def __init__(self, in_dict=None): + """Initialize the class and all nested dictionaries.""" + if in_dict is None: + in_dict = {} + for key, value in in_dict.items(): + if isinstance(value, dict): + setattr(self, key, MockStub(value)) + elif type(value) in (list, tuple): + setattr( + self, + key, + [MockStub(x) if isinstance(x, dict) else x for x in value], + ) + else: + setattr(self, key, value) + super().__init__(in_dict) + + def __getattr__(self, name): + """Fallback to returning self if attribute doesn't exist.""" + return self + + def __getitem__(self, key): + """Get an item from the dictionary-like object. + + If the key is a string, this method will attempt to get an attribute with that name. + If the key is not found, this method will return the object itself. + """ + if isinstance(key, str): + item = getattr(self, key, self) + try: + item = super().__getitem__(key) + except KeyError: + item = self + return item + + def __call__(self, *args, **kwargs): + """Allow MockStub to be used like a function.""" + return self diff --git a/manifester/logger.py b/manifester/logger.py index ef85224..3926201 100644 --- a/manifester/logger.py +++ b/manifester/logger.py @@ -1,3 +1,4 @@ +"""Defines manifester's internal logging.""" import logging from pathlib import Path @@ -7,26 +8,22 @@ def setup_logzero(level="info", path="logs/manifester.log", silent=True): + """Call logzero setup with the given settings.""" Path(path).parent.mkdir(parents=True, exist_ok=True) log_fmt = "%(color)s[%(levelname)s %(asctime)s]%(end_color)s %(message)s" debug_fmt = ( - "%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]" - "%(end_color)s %(message)s" + "%(color)s[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]%(end_color)s %(message)s" ) log_level = getattr(logging, level.upper(), logging.INFO) # formatter for terminal - formatter = logzero.LogFormatter( - fmt=debug_fmt if log_level is logging.DEBUG else log_fmt - ) + formatter = logzero.LogFormatter(fmt=debug_fmt if log_level is logging.DEBUG else log_fmt) logzero.setup_default_logger(formatter=formatter, disableStderrLogger=silent) logzero.loglevel(log_level) # formatter for file formatter = logzero.LogFormatter( fmt=debug_fmt if log_level is logging.DEBUG else log_fmt, color=False ) - logzero.logfile( - path, loglevel=log_level, maxBytes=1e9, backupCount=3, formatter=formatter - ) + logzero.logfile(path, loglevel=log_level, maxBytes=1e9, backupCount=3, formatter=formatter) setup_logzero(level=settings.get("log_level", "info")) diff --git a/manifester/manifester.py b/manifester/manifester.py index 5cc3747..46f7293 100644 --- a/manifester/manifester.py +++ b/manifester/manifester.py @@ -1,28 +1,39 @@ -import random -import string -from dynaconf.utils.boxing import DynaBox +"""Main interface for the RHSM API. + +This module defines the `Manifester` class, which provides methods for authenticating to and +interacting with the RHSM Subscription API for the purpose of generating a subscription manifest. +""" from functools import cached_property from pathlib import Path +import random +import string -import requests +from dynaconf.utils.boxing import DynaBox from logzero import logger +from requests.exceptions import Timeout -from manifester.helpers import simple_retry -from manifester.helpers import process_sat_version -from manifester.logger import setup_logzero +from manifester.helpers import process_sat_version, simple_retry from manifester.settings import settings class Manifester: + """Main Manifester class responsible for generating a manifest from the provided settings.""" + def __init__(self, manifest_category, allocation_name=None, **kwargs): if isinstance(manifest_category, dict): self.manifest_data = DynaBox(manifest_category) else: self.manifest_data = settings.manifest_category.get(manifest_category) - self.allocation_name = allocation_name or "".join( - random.sample(string.ascii_letters, 10) - ) - self.manifest_name = Path(f'{self.allocation_name}_manifest.zip') + if kwargs.get("requester") is not None: + self.requester = kwargs["requester"] + self.is_mock = True + else: + import requests + + self.requester = requests + self.is_mock = False + self.allocation_name = allocation_name or "".join(random.sample(string.ascii_letters, 10)) + self.manifest_name = Path(f"{self.allocation_name}_manifest.zip") self.offline_token = kwargs.get("offline_token", self.manifest_data.offline_token) self.subscription_data = self.manifest_data.subscription_data self.token_request_data = { @@ -33,10 +44,11 @@ def __init__(self, manifest_category, allocation_name=None, **kwargs): self.simple_content_access = kwargs.get( "simple_content_access", self.manifest_data.simple_content_access ) - self.token_request_url = self.manifest_data.get("url", {}).get("token_request", settings.url.token_request) - self.allocations_url = self.manifest_data.get("url", {}).get("allocations", settings.url.allocations) + self.token_request_url = self.manifest_data.get("url").get("token_request") + self.allocations_url = self.manifest_data.get("url").get("allocations") self._access_token = None self._subscription_pools = None + self._active_pools = [] self.sat_version = process_sat_version( kwargs.get("sat_version", self.manifest_data.sat_version), self.valid_sat_versions, @@ -44,39 +56,46 @@ def __init__(self, manifest_category, allocation_name=None, **kwargs): @property def access_token(self): + """Representation of an RHSM API access token. + + Used to authenticate requests to the RHSM API. + """ if not self._access_token: token_request_data = {"data": self.token_request_data} logger.debug("Generating access token") token_data = simple_retry( - requests.post, + self.requester.post, cmd_args=[f"{self.token_request_url}"], cmd_kwargs=token_request_data, ).json() - self._access_token = token_data["access_token"] + if self.is_mock: + self._access_token = token_data.access_token + else: + self._access_token = token_data["access_token"] return self._access_token - + @cached_property def valid_sat_versions(self): + """Retrieves the list of valid Satellite versions from the RHSM API.""" headers = { "headers": {"Authorization": f"Bearer {self.access_token}"}, - "proxies": self.manifest_data.get("proxies", settings.proxies), + "proxies": self.manifest_data.get("proxies"), } - valid_sat_versions = [] sat_versions_response = simple_retry( - requests.get, - cmd_args=[ - f"{self.allocations_url}/versions" - ], + self.requester.get, + cmd_args=[f"{self.allocations_url}/versions"], cmd_kwargs=headers, - ).json() - for ver_dict in sat_versions_response["body"]: - valid_sat_versions.append(ver_dict["value"]) + ).json() + if self.is_mock: + sat_versions_response = sat_versions_response.version_response + valid_sat_versions = [ver_dict["value"] for ver_dict in sat_versions_response["body"]] return valid_sat_versions def create_subscription_allocation(self): + """Creates a new consumer in the provided RHSM account and returns its UUID.""" allocation_data = { "headers": {"Authorization": f"Bearer {self.access_token}"}, - "proxies": self.manifest_data.get("proxies", settings.proxies), + "proxies": self.manifest_data.get("proxies"), "params": { "name": f"{self.allocation_name}", "version": f"{self.sat_version}", @@ -84,28 +103,19 @@ def create_subscription_allocation(self): }, } self.allocation = simple_retry( - requests.post, + self.requester.post, cmd_args=[f"{self.allocations_url}"], cmd_kwargs=allocation_data, ).json() - logger.debug( - f"Received response {self.allocation} when attempting to create allocation." - ) - if ("error" in self.allocation.keys() and - "invalid version" in self.allocation['error'].values()): - raise ValueError( - f"{self.sat_version} is not a valid version number." - "Versions must be in the form of \"sat-X.Y\". Current" - f"valid versions are {self.valid_sat_versions}." - ) + logger.debug(f"Received response {self.allocation} when attempting to create allocation.") self.allocation_uuid = self.allocation["body"]["uuid"] if self.simple_content_access == "disabled": simple_retry( - requests.put, + self.requester.put, cmd_args=[f"{self.allocations_url}/{self.allocation_uuid}"], cmd_kwargs={ "headers": {"Authorization": f"Bearer {self.access_token}"}, - "proxies": self.manifest_data.get("proxies", settings.proxies), + "proxies": self.manifest_data.get("proxies"), "json": {"simpleContentAccess": "disabled"}, }, ) @@ -116,14 +126,17 @@ def create_subscription_allocation(self): return self.allocation_uuid def delete_subscription_allocation(self): + """Deletes the specified subscription allocation and returns the RHSM API's response.""" self._access_token = None data = { "headers": {"Authorization": f"Bearer {self.access_token}"}, - "proxies": self.manifest_data.get("proxies", settings.proxies), + "proxies": self.manifest_data.get("proxies"), "params": {"force": "true"}, } + if self.is_mock: + self.allocation_uuid = self.allocation_uuid.uuid response = simple_retry( - requests.delete, + self.requester.delete, cmd_args=[f"{self.allocations_url}/{self.allocation_uuid}"], cmd_kwargs=data, ) @@ -131,42 +144,45 @@ def delete_subscription_allocation(self): @property def subscription_pools(self): + """Fetches the list of subscription pools from account. + + Returns a list of dictionaries containing metadata from the pools. + """ + MAX_RESULTS_PER_PAGE = 50 if not self._subscription_pools: _offset = 0 data = { "headers": {"Authorization": f"Bearer {self.access_token}"}, - "proxies": self.manifest_data.get("proxies", settings.proxies), + "proxies": self.manifest_data.get("proxies"), "params": {"offset": _offset}, } self._subscription_pools = simple_retry( - requests.get, - cmd_args=[ - f"{self.allocations_url}/{self.allocation_uuid}/pools" - ], + self.requester.get, + cmd_args=[f"{self.allocations_url}/{self.allocation_uuid}/pools"], cmd_kwargs=data, ).json() + if self.is_mock: + self._subscription_pools = self._subscription_pools.pool_response _results = len(self._subscription_pools["body"]) # The endpoint used in the above API call can return a maximum of 50 subscription pools. # For organizations with more than 50 subscription pools, the loop below works around # this limit by repeating calls with a progressively larger value for the `offset` # parameter. - while _results == 50: + while _results == MAX_RESULTS_PER_PAGE: _offset += 50 - logger.debug( - f"Fetching additional subscription pools with an offset of {_offset}." - ) + logger.debug(f"Fetching additional subscription pools with an offset of {_offset}.") data = { "headers": {"Authorization": f"Bearer {self.access_token}"}, - "proxies": self.manifest_data.get("proxies", settings.proxies), + "proxies": self.manifest_data.get("proxies"), "params": {"offset": _offset}, } offset_pools = simple_retry( - requests.get, - cmd_args=[ - f"{self.allocations_url}/{self.allocation_uuid}/pools" - ], + self.requester.get, + cmd_args=[f"{self.allocations_url}/{self.allocation_uuid}/pools"], cmd_kwargs=data, ).json() + if self.is_mock: + offset_pools = offset_pools.pool_response self._subscription_pools["body"] += offset_pools["body"] _results = len(offset_pools["body"]) total_pools = len(self._subscription_pools["body"]) @@ -176,31 +192,29 @@ def subscription_pools(self): return self._subscription_pools def add_entitlements_to_allocation(self, pool_id, entitlement_quantity): + """Attempts to add the set of subscriptions defined in the settings to the allocation.""" data = { "headers": {"Authorization": f"Bearer {self.access_token}"}, - "proxies": self.manifest_data.get("proxies", settings.proxies), + "proxies": self.manifest_data.get("proxies"), "params": {"pool": f"{pool_id}", "quantity": f"{entitlement_quantity}"}, } add_entitlements = simple_retry( - requests.post, - cmd_args=[ - f"{self.allocations_url}/{self.allocation_uuid}/entitlements" - ], + self.requester.post, + cmd_args=[f"{self.allocations_url}/{self.allocation_uuid}/entitlements"], cmd_kwargs=data, ) return add_entitlements def verify_allocation_entitlements(self, entitlement_quantity, subscription_name): - logger.info( - f"Verifying the entitlement quantity of {subscription_name} on the allocation." - ) + """Checks that the entitlements in the allocation match those defined in settings.""" + logger.info(f"Verifying the entitlement quantity of {subscription_name} on the allocation.") data = { "headers": {"Authorization": f"Bearer {self.access_token}"}, - "proxies": self.manifest_data.get("proxies", settings.proxies), + "proxies": self.manifest_data.get("proxies"), "params": {"include": "entitlements"}, } self.entitlement_data = simple_retry( - requests.get, + self.requester.get, cmd_args=[f"{self.allocations_url}/{self.allocation_uuid}"], cmd_kwargs=data, ).json() @@ -214,9 +228,7 @@ def verify_allocation_entitlements(self, entitlement_quantity, subscription_name logger.debug(f"Current entitlement is {current_entitlement}") self.attached_quantity = current_entitlement[0]["entitlementQuantity"] if self.attached_quantity == entitlement_quantity: - logger.debug( - f"Operation successful. Attached {self.attached_quantity} entitlements." - ) + logger.debug(f"Operation successful. Attached {self.attached_quantity} entitlements.") return True elif self.attached_quantity < entitlement_quantity: logger.debug( @@ -231,18 +243,24 @@ def verify_allocation_entitlements(self, entitlement_quantity, subscription_name return True def process_subscription_pools(self, subscription_pools, subscription_data): + """Loops through the list of subscription pools in the account. + + Identifies pools that match the subscription names and quantities defined in settings, then + attempts to add the specified quantity of each subscription to the allocation. + """ + SUCCESS_CODE = 200 logger.debug(f"Finding a matching pool for {subscription_data['name']}.") matching = [ d for d in subscription_pools["body"] if d["subscriptionName"] == subscription_data["name"] ] - logger.debug( - f"The following pools are matches for this subscription: {matching}" - ) + logger.debug(f"The following pools are matches for this subscription: {matching}") for match in matching: - if (match["entitlementsAvailable"] > subscription_data["quantity"] or - match["entitlementsAvailable"] == -1): + if ( + match["entitlementsAvailable"] > subscription_data["quantity"] + or match["entitlementsAvailable"] == -1 + ): logger.debug( f"Pool {match['id']} is a match for this subscription and has " f"{match['entitlementsAvailable']} entitlements available." @@ -251,7 +269,7 @@ def process_subscription_pools(self, subscription_pools, subscription_data): pool_id=match["id"], entitlement_quantity=subscription_data["quantity"], ) - # if the above is using simple_rety, it will raise an exception + # if the above is using simple_retry, it will raise an exception # and never trigger the following block if add_entitlements.status_code in [404, 429, 500, 504]: verify_entitlements = self.verify_allocation_entitlements( @@ -288,73 +306,78 @@ def process_subscription_pools(self, subscription_pools, subscription_data): f"Successfully added {subscription_data['quantity']} entitlements of " f"{subscription_data['name']} to the allocation." ) + self._active_pools.append(match) break - elif add_entitlements.status_code == 200: + elif add_entitlements.status_code == SUCCESS_CODE: logger.debug( f"Successfully added {subscription_data['quantity']} entitlements of " f"{subscription_data['name']} to the allocation." ) + self._active_pools.append(match) break else: - raise Exception( + raise RuntimeError( "Something went wrong while adding entitlements. Received response status " f"{add_entitlements.status_code}." ) def trigger_manifest_export(self): + """Triggers job to export manifest from subscription allocation. + + Starts the export job, monitors the status of the job, and downloads the manifest on + successful completion of the job. + """ + MAX_REQUESTS = 50 + SUCCESS_CODE = 200 data = { "headers": {"Authorization": f"Bearer {self.access_token}"}, - "proxies": self.manifest_data.get("proxies", settings.proxies), + "proxies": self.manifest_data.get("proxies"), } - # Should this use the XDG Base Directory Specification? local_file = Path(f"manifests/{self.manifest_name}") local_file.parent.mkdir(parents=True, exist_ok=True) logger.info( f"Triggering manifest export job for subscription allocation {self.allocation_name}" ) trigger_export_job = simple_retry( - requests.get, - cmd_args=[ - f"{self.allocations_url}/{self.allocation_uuid}/export" - ], + self.requester.get, + cmd_args=[f"{self.allocations_url}/{self.allocation_uuid}/export"], cmd_kwargs=data, ).json() export_job_id = trigger_export_job["body"]["exportJobID"] export_job = simple_retry( - requests.get, - cmd_args=[ - f"{self.allocations_url}/{self.allocation_uuid}/exportJob/{export_job_id}" - ], + self.requester.get, + cmd_args=[f"{self.allocations_url}/{self.allocation_uuid}/exportJob/{export_job_id}"], cmd_kwargs=data, ) request_count = 1 limit_exceeded = False - while export_job.status_code != 200: + while export_job.status_code != SUCCESS_CODE: export_job = simple_retry( - requests.get, + self.requester.get, cmd_args=[ f"{self.allocations_url}/{self.allocation_uuid}/exportJob/{export_job_id}" ], cmd_kwargs=data, ) - logger.debug( - f"Attempting to export manifest. Attempt number: {request_count}" - ) - if request_count > 50: + logger.debug(f"Attempting to export manifest. Attempt number: {request_count}") + if request_count > MAX_REQUESTS: limit_exceeded = True logger.info( "Manifest export job status check limit exceeded. This may indicate an " "upstream issue with Red Hat Subscription Management." ) - break + raise Timeout("Export timeout exceeded") request_count += 1 if limit_exceeded: self.content = None return self export_job = export_job.json() - export_href = export_job["body"]["href"] + if self.is_mock: + export_href = export_job.body["href"] + else: + export_href = export_job["body"]["href"] manifest = simple_retry( - requests.get, + self.requester.get, cmd_args=[f"{export_href}"], cmd_kwargs=data, ) @@ -368,6 +391,11 @@ def trigger_manifest_export(self): return manifest def get_manifest(self): + """Provides a subscription manifest based on settings. + + Calls the methods required to create a new subscription allocation, add the appropriate + subscriptions to the allocation, export a manifest, and download the manifest. + """ self.create_subscription_allocation() for sub in self.subscription_data: self.process_subscription_pools( @@ -377,6 +405,7 @@ def get_manifest(self): return self.trigger_manifest_export() def __enter__(self): + """Generates and returns a manifest.""" try: return self.get_manifest() except: @@ -384,4 +413,5 @@ def __enter__(self): raise def __exit__(self, *tb_args): + """Deletes subscription allocation on teardown.""" self.delete_subscription_allocation() diff --git a/manifester/settings.py b/manifester/settings.py index 37b6712..ba83d1f 100644 --- a/manifester/settings.py +++ b/manifester/settings.py @@ -1,8 +1,8 @@ +"""Retrieves settings from configuration file and runs Dynaconf validators.""" import os from pathlib import Path -from dynaconf import Dynaconf -from dynaconf import Validator +from dynaconf import Dynaconf, Validator settings_file = "manifester_settings.yaml" MANIFESTER_DIRECTORY = Path() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6636938 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,183 @@ +[tool.black] +line-length = 100 +skip-string-normalization = true +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.hg + | \.mypy_cache + | \.venv + | _build + | buck-out + | build + | dist +)/ +''' +[build-system] +requires = ["setuptools", "setuptools-scm[toml]", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "manifester" +description = "Red Hat subscriptions made manifest." +readme = "README.md" +requires-python = ">=3.10" +license = {file = "LICENSE", name = "Apache License Version 2.0"} +keywords = ["manifester", "RHSM"] +authors = [ + {name = "Danny Synk", email = "dsynk@redhat.com"} +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "click", + "dynaconf", + "logzero", + "pytest", + "pyyaml", + "requests", + "setuptools", +] +dynamic = ["version"] + +[project.urls] +Repository = "https://github.com/SatelliteQE/manifester" + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "ruff", +] +setup = [ + "build", + "twine", +] + +[tools.setuptools] +platforms = ["any"] +zip-safe = false +include-package-data = true + +[tool.setuptools.packages.find] +include = ["manifester"] + +[tool.setuptools_scm] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 +target-version = "py311" +fixable = ["ALL"] + +select = [ + "B002", # Python does not support the unary prefix increment + "B007", # Loop control variable {name} not used within loop body + "B009", # Do not call getattr with a constant attribute value + "B010", # Do not call setattr with a constant attribute value + "B011", # Do not `assert False`, raise `AssertionError` instead + "B013", # Redundant tuple in exception handler + "B014", # Exception handler with duplicate exception + "B023", # Function definition does not bind loop variable {name} + "B026", # Star-arg unpacking after a keyword argument is strongly discouraged + "BLE001", # Using bare except clauses is prohibited + "C", # complexity + "C4", # flake8-comprehensions + "COM818", # Trailing comma on bare tuple prohibited + "D", # docstrings + "E", # pycodestyle + "F", # pyflakes/autoflake + "G", # flake8-logging-format + "I", # isort + "ISC001", # Implicitly concatenated string literals on one line + "N804", # First argument of a class method should be named cls + "N805", # First argument of a method should be named self + "N815", # Variable {name} in class scope should not be mixedCase + "N999", # Invalid module name: '{name}' + "PERF", # Perflint rules + "PGH004", # Use specific rule codes when using noqa + "PLC0414", # Useless import alias. Import alias does not rename original package. + "PLC", # pylint + "PLE", # pylint + "PLR", # pylint + "PLW", # pylint + "PTH", # Use pathlib + "RUF", # Ruff-specific rules + "S103", # bad-file-permissions + "S108", # hardcoded-temp-file + "S110", # try-except-pass + "S112", # try-except-continue + "S113", # Probable use of requests call without timeout + "S306", # suspicious-mktemp-usage + "S307", # suspicious-eval-usage + "S601", # paramiko-call + "S602", # subprocess-popen-with-shell-equals-true + "S604", # call-with-shell-equals-true + "S609", # unix-command-wildcard-injection + "SIM105", # Use contextlib.suppress({exception}) instead of try-except-pass + "SIM117", # Merge with-statements that use the same scope + "SIM118", # Use {key} in {dict} instead of {key} in {dict}.keys() + "SIM201", # Use {left} != {right} instead of not {left} == {right} + "SIM208", # Use {expr} instead of not (not {expr}) + "SIM212", # Use {a} if {a} else {b} instead of {b} if not {a} else {a} + "SIM300", # Yoda conditions. Use 'age == 42' instead of '42 == age'. + "SIM401", # Use get from dict with default instead of an if block + "T100", # Trace found: {name} used + "T20", # flake8-print + "TRY004", # Prefer TypeError exception for invalid type + "TRY200", # Use raise from to specify exception cause + "TRY302", # Remove exception handler; error is immediately re-raised + "PLR0911", # Too many return statements ({returns} > {max_returns}) + "PLR0912", # Too many branches ({branches} > {max_branches}) + "PLR0915", # Too many statements ({statements} > {max_statements}) + "PLR2004", # Magic value used in comparison, consider replacing {value} with a constant variable + "PLW2901", # Outer {outer_kind} variable {name} overwritten by inner {inner_kind} target + "UP", # pyupgrade + "W", # pycodestyle +] + +ignore = [ + "ANN", # flake8-annotations + "PGH001", # No builtin eval() allowed + "D203", # 1 blank line required before class docstring + "D213", # Multi-line docstring summary should start at the second line + "D406", # Section name should end with a newline + "D407", # Section name underlining + "E731", # do not assign a lambda expression, use a def + "PLR0913", # Too many arguments to function call ({c_args} > {max_args}) + "RUF012", # Mutable class attributes should be annotated with typing.ClassVar + "D107", # Missing docstring in __init__ +] + +[tool.ruff.per-file-ignores] +"manifester/__init__.py" = ["D104", "F401",] +"manifester/manifester.py" = ["D401",] +"tests/test_manifester.py" = ["D100", "E501", "PLR0911", "PLR2004",] + +[tool.ruff.isort] +force-sort-within-sections = true +known-first-party = [ + "manifester", +] +combine-as-imports = true + +[tool.ruff.flake8-pytest-style] +fixture-parentheses = false +mark-parentheses = false + +[tool.ruff.flake8-quotes] +inline-quotes = "single" + +[tool.ruff.mccabe] +max-complexity = 20 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 752caee..0000000 --- a/setup.cfg +++ /dev/null @@ -1,42 +0,0 @@ -[metadata] -name = manifester -description = Manifester dynamically generates subscription manifests using the Red Hat Subscription Managament API. -long_description = file: README.md -long_description_content_type = text/markdown -author = Danny Synk -author_email = dsynk@redhat.com -url = https://github.com/SatelliteQE/manifester -license = Apache -keywords = rhsm, red hat -classifiers = - Development Status :: 3 - Alpha - Intended Audience :: Developers - License :: OSI Approved :: GNU General Public License v3 (GPLv3) - Natural Language :: English - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - -[options] -install_requires = - click - dynaconf>=3.1.0 - logzero - pyyaml - requests - setuptools -packages = find: -zip_safe = False - -[options.extras_require] -test = pytest -setup = - setuptools - setuptools-scm - wheel - twine - -[options.entry_points] -console_scripts = - manifester = manifester.commands:cli diff --git a/setup.py b/setup.py deleted file mode 100644 index 42eaec6..0000000 --- a/setup.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup - - -setup( - use_scm_version=True, -) diff --git a/tests/test_manifester.py b/tests/test_manifester.py new file mode 100644 index 0000000..b9145a8 --- /dev/null +++ b/tests/test_manifester.py @@ -0,0 +1,242 @@ +from functools import cached_property +import random +import string +import uuid + +import pytest +from requests.exceptions import Timeout + +from manifester import Manifester +from manifester.helpers import MockStub, fake_http_response_code + +manifest_data = { + "log_level": "debug", + "offline_token": "test", + "proxies": {"https": ""}, + "url": { + "token_request": "https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token", + "allocations": "https://api.access.redhat.com/management/v1/allocations", + }, + "sat_version": "sat-6.14", + "subscription_data": [ + { + "name": "Red Hat Enterprise Linux Server, Premium (Physical or Virtual Nodes)", + "quantity": 1, + }, + { + "name": "Red Hat Satellite Infrastructure Subscription", + "quantity": 1, + }, + { + "name": "Red Hat Beta Access", + "quantity": 1, + }, + { + "name": "Red Hat Enterprise Linux for Virtual Datacenters, Premium", + "quantity": 1, + }, + ], + "simple_content_access": "enabled", +} + +sub_pool_response = { + "body": [ + { + "id": f"{uuid.uuid4().hex}", + "subscriptionName": "Red Hat Satellite Infrastructure Subscription", + "entitlementsAvailable": 8, + }, + { + "id": f"{uuid.uuid4().hex}", + "subscriptionName": "Red Hat Enterprise Linux for Virtual Datacenters, Premium", + "entitlementsAvailable": 8, + }, + { + "id": f"{uuid.uuid4().hex}", + "subscriptionName": "Red Hat Beta Access", + "entitlementsAvailable": 8, + }, + { + "id": f"{uuid.uuid4().hex}", + "subscriptionName": "Red Hat Enterprise Linux Server, Premium (Physical or Virtual Nodes)", + "entitlementsAvailable": 8, + }, + ], +} + + +class RhsmApiStub(MockStub): + """Returns mock responses for RHSM API endpoints related to creating manifests.""" + + def __init__(self, in_dict=None, **kwargs): + self._good_codes = kwargs.get("good_codes", [200]) + self._bad_codes = kwargs.get("bad_codes", [429, 500, 504]) + self._fail_rate = kwargs.get("fail_rate", 0) + self._has_offset = kwargs.get("has_offset", False) + super().__init__(in_dict) + + @cached_property + def status_code(self): + """HTTP response code of current request.""" + return fake_http_response_code(self._good_codes, self._bad_codes, self._fail_rate) + + def post(self, *args, **kwargs): + """Simulate responses to POST requests for RHSM API endpoints used by Manifester.""" + if args[0].endswith("openid-connect/token"): + self.access_token = "this is a simulated access token" + return self + if args[0].endswith("allocations"): + self.uuid = "1234567890" + return self + if args[0].endswith("entitlements"): + self.params = kwargs["params"] + return self + + def get(self, *args, **kwargs): + """Simulate responses to GET requests for RHSM API endpoints used by Manifester.""" + if args[0].endswith("versions"): + del self.status_code + self.version_response = { + "body": [ + {"value": "sat-6.14", "description": "Satellite 6.14"}, + {"value": "sat-6.13", "description": "Satellite 6.13"}, + {"value": "sat-6.12", "description": "Satellite 6.12"}, + ] + } + return self + if args[0].endswith("pools") and not self._has_offset: + self.pool_response = sub_pool_response + return self + if args[0].endswith("pools") and self._has_offset: + if kwargs["params"]["offset"] != 50: + self.pool_response = {"body": []} + for _x in range(50): + self.pool_response["body"].append( + { + "id": f'{"".join(random.sample(string.ascii_letters, 12))}', + "subscriptionName": "Red Hat Satellite Infrastructure Subscription", + "entitlementsAvailable": random.randrange(100), + } + ) + return self + else: + self.pool_response["body"] += sub_pool_response["body"] + return self + if "allocations" in args[0] and not ("export" in args[0] or "pools" in args[0]): + self.allocation_data = "this allocation data also includes entitlement data" + return self + if args[0].endswith("export"): + self.body = {"exportJobID": "123456", "href": "exportJob"} + return self + if "exportJob" in args[0]: + del self.status_code + if self.force_export_failure is True and not self._has_offset: + self._good_codes = [202] + else: + self._good_codes = [202, 200] + self.body = {"exportID": 27, "href": "https://example.com/export/98ef892ac11"} + return self + if "export" in args[0] and not args[0].endswith("export"): + del self.status_code + self._good_codes = [200] + # Manifester expects a bytes-type object to be returned as the manifest + self.content = b"this is a simulated manifest" + return self + + def delete(self, *args, **kwargs): + """Simulate responses to DELETE requests for RHSM API endpoints used by Manifester.""" + if args[0].endswith("allocations/1234567890") and kwargs["params"]["force"] == "true": + del self.status_code + self.content = b"" + self._good_codes = [204] + return self + + +def test_basic_init(): + """Test that manifester can initialize with the minimum required arguments.""" + manifester_inst = Manifester( + manifest_category=manifest_data, requester=RhsmApiStub(in_dict=None) + ) + assert isinstance(manifester_inst, Manifester) + assert manifester_inst.access_token == "this is a simulated access token" + + +def test_create_allocation(): + """Test that manifester's create_subscription_allocation method returns a UUID.""" + manifester = Manifester(manifest_category=manifest_data, requester=RhsmApiStub(in_dict=None)) + allocation_uuid = manifester.create_subscription_allocation() + assert allocation_uuid.uuid == "1234567890" + + +def test_negative_simple_retry_timeout(): + """Test that exceeding the attempt limit when retrying a failed API call results in an exception.""" + manifester = Manifester( + manifest_category=manifest_data, requester=RhsmApiStub(in_dict=None, fail_rate=0) + ) + manifester.requester._fail_rate = 100 + with pytest.raises(Exception) as exception: + manifester.get_manifest() + assert str(exception.value) == "Retry timeout exceeded" + + +def test_negative_manifest_export_timeout(): + """Test that exceeding the attempt limit when exporting a manifest results in an exception.""" + manifester = Manifester( + manifest_category=manifest_data, + requester=RhsmApiStub(in_dict={"force_export_failure": True}), + ) + with pytest.raises(Timeout) as exception: + manifester.get_manifest() + assert str(exception.value) == "Export timeout exceeded" + + +def test_get_manifest(): + """Test that manifester's get_manifest method returns a manifest.""" + manifester = Manifester(manifest_category=manifest_data, requester=RhsmApiStub(in_dict=None)) + manifest = manifester.get_manifest() + assert manifest.content.decode("utf-8") == "this is a simulated manifest" + assert manifest.status_code == 200 + + +def test_delete_subscription_allocation(): + """Test that manifester's delete_subscription_allocation method deletes a subscription allocation.""" + manifester = Manifester(manifest_category=manifest_data, requester=RhsmApiStub(in_dict=None)) + manifester.get_manifest() + response = manifester.delete_subscription_allocation() + assert response.status_code == 204 + assert response.content == b"" + + +def test_ingest_manifest_data_via_dict(): + """Test that manifester is able to read configuration data from a dictionary.""" + manifester = Manifester(manifest_category=manifest_data, requester=RhsmApiStub(in_dict=None)) + assert manifester.subscription_data == manifest_data["subscription_data"] + assert manifester.simple_content_access == manifest_data["simple_content_access"] + assert manifester.token_request_url == manifest_data["url"]["token_request"] + assert manifester.allocations_url == manifest_data["url"]["allocations"] + assert manifester.sat_version == manifest_data["sat_version"] + + +def test_get_subscription_pools_with_offset(): + """Tests that manifester can retrieve all pools from an account containing more than 50 pools.""" + manifester = Manifester( + manifest_category=manifest_data, requester=RhsmApiStub(in_dict=None, has_offset=True) + ) + manifester.get_manifest() + assert len(manifester.subscription_pools["body"]) > 50 + + +def test_correct_subs_added_to_allocation(): + """Test that subs added to the allocation match the subscription data in manifester's config.""" + manifester = Manifester(manifest_category=manifest_data, requester=RhsmApiStub(in_dict=None)) + manifester.get_manifest() + active_subs = sorted([x["subscriptionName"] for x in manifester._active_pools]) + sub_names_from_config = sorted([x["NAME"] for x in manifester.subscription_data]) + assert active_subs == sub_names_from_config + + +def test_invalid_sat_version(): + """Test that an invalid sat_version value will be replaced with the latest valid sat_version.""" + manifest_data["sat_version"] = "sat-6.20" + manifester = Manifester(manifest_category=manifest_data, requester=RhsmApiStub(in_dict=None)) + assert manifester.sat_version == "sat-6.14"