From dbefc22fdc79852905c8d549b1568fc6519b8f88 Mon Sep 17 00:00:00 2001 From: Jacob Callahan Date: Fri, 16 Feb 2024 11:24:26 -0500 Subject: [PATCH] [POC] New-Style Upgrade Tests SharedResource: - Added the ability to validate the result of a given action function via an action_validator function. - Made an improvement to exiting under error conditions that improved tracking file cleanup. New directory for new-style upgrades located at tests/new_upgrades. This will help to keep changes isolated from the existing upgrade tests. new_upgrades/conftest: - Removed the requirement for all upgrade tests to be marked as pre/post - Introduced fixtures that coordinate checkout/checkin actions between multiple xdist workers. - Introduced a fixture that performs an upgrade on a target satellite - Introduced a fixture that is used for two test conversions in different modules. test conversions: - test_cv_upgrade_scenario and test_scenario_custom_repo_check converted - pre-upgrade tests are now pre-upgrade fixtures that perform setup and yield their data in Box objects instead of saving to disk - post-upgrade tests can now directly access the setup objects by inheriting the pre-upgrade fixture results settings: - Added SATELLITE_DEPLOY_WORKFLOW and SATELLITE_UPGRADE_JOB_TEMPLATE to upgrade.yaml --- conf/upgrade.yaml.template | 4 + requirements.txt | 13 +-- robottelo/utils/shared_resource.py | 32 +++++- tests/new_upgrades/__init__.py | 0 tests/new_upgrades/conftest.py | 98 +++++++++++++++++++ tests/new_upgrades/test_contentview.py | 124 ++++++++++++++++++++++++ tests/new_upgrades/test_repository.py | 117 ++++++++++++++++++++++ tests/robottelo/test_shared_resource.py | 10 +- 8 files changed, 383 insertions(+), 15 deletions(-) create mode 100644 tests/new_upgrades/__init__.py create mode 100644 tests/new_upgrades/conftest.py create mode 100644 tests/new_upgrades/test_contentview.py create mode 100644 tests/new_upgrades/test_repository.py diff --git a/conf/upgrade.yaml.template b/conf/upgrade.yaml.template index 1b971ba1b30..b8113843e5f 100644 --- a/conf/upgrade.yaml.template +++ b/conf/upgrade.yaml.template @@ -5,6 +5,10 @@ UPGRADE: TO_VERSION: "6.9" # Satellite, Capsule hosts RHEL operating system version. OS: "rhel7" + # The workflow Broker should use to checkout a to-be-upgraded Satellite + SATELLITE_DEPLOY_WORKFLOW: deploy-satellite-upgrade + # The job template Broker should use to upgrade a Satellite + SATELLITE_UPGRADE_JOB_TEMPLATE: satellite-upgrade # Capsule's activation key will only be available when we spawn the VM using upgrade template. CAPSULE_AK: RHEL6: diff --git a/requirements.txt b/requirements.txt index 599d6fa5f3f..ff96f06c25a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,14 @@ # Version updates managed by dependabot betelgeuse==1.11.0 -# broker[docker]==0.4.1 - Temporarily disabled, see below +broker[docker]==0.4.8 cryptography==42.0.5 deepdiff==6.7.1 -docker==7.0.0 # Temporary until Broker is back on PyPi dynaconf[vault]==3.2.4 fauxfactory==3.1.0 jinja2==3.1.3 manifester==0.0.14 navmazing==1.2.2 -paramiko==3.4.0 # Temporary until Broker is back on PyPi productmd==1.38 pyotp==2.9.0 python-box==7.1.1 @@ -30,11 +28,6 @@ wait-for==1.2.0 wrapanapi==3.6.0 # Get airgun, nailgun and upgrade from master -git+https://github.com/SatelliteQE/airgun.git@master#egg=airgun -git+https://github.com/SatelliteQE/nailgun.git@master#egg=nailgun -# Broker currently is unable to push to PyPi due to [1] and [2] -# In the meantime, we install directly from the repo -# [1] - https://github.com/ParallelSSH/ssh2-python/issues/193 -# [2] - https://github.com/pypi/warehouse/issues/7136 -git+https://github.com/SatelliteQE/broker.git@0.4.7#egg=broker +airgun @ git+https://github.com/SatelliteQE/airgun.git@master#egg=airgun +nailgun @ git+https://github.com/SatelliteQE/nailgun.git@master#egg=nailgun --editable . diff --git a/robottelo/utils/shared_resource.py b/robottelo/utils/shared_resource.py index 0ad0bd92e46..9b36c4f0d52 100644 --- a/robottelo/utils/shared_resource.py +++ b/robottelo/utils/shared_resource.py @@ -29,6 +29,10 @@ from broker.helpers import FileLock +class SharedResourceError(Exception): + """An exception class for SharedResource errors.""" + + class SharedResource: """A class representing a shared resource. @@ -43,19 +47,21 @@ class SharedResource: is_recovering (bool): Whether the current instance is recovering from an error or not. """ - def __init__(self, resource_name, action, *action_args, **action_kwargs): + def __init__(self, resource_name, action, *action_args, action_validator=None, **action_kwargs): """Initializes a new instance of the SharedResource class. Args: resource_name (str): The name of the shared resource. action (function): The function to be executed when the resource is ready. action_args (tuple): The arguments to be passed to the action function. + action_validator (function): The function to validate the action results. action_kwargs (dict): The keyword arguments to be passed to the action function. """ self.resource_file = Path(f"/tmp/{resource_name}.shared") self.lock_file = FileLock(self.resource_file) self.id = str(uuid4().fields[-1]) self.action = action + self.action_validator = action_validator self.action_is_recoverable = action_kwargs.pop("action_is_recoverable", False) self.action_args = action_args self.action_kwargs = action_kwargs @@ -151,6 +157,14 @@ def register(self): curr_data["statuses"][self.id] = "pending" self.resource_file.write_text(json.dumps(curr_data, indent=4)) + def unregister(self): + """Unregisters the current process as a watcher.""" + with self.lock_file: + curr_data = json.loads(self.resource_file.read_text()) + curr_data["watchers"].remove(self.id) + del curr_data["statuses"][self.id] + self.resource_file.write_text(json.dumps(curr_data, indent=4)) + def ready(self): """Marks the current process as ready to perform the action.""" self._update_status("ready") @@ -163,10 +177,15 @@ def done(self): def act(self): """Attempt to perform the action.""" try: - self.action(*self.action_args, **self.action_kwargs) + result = self.action(*self.action_args, **self.action_kwargs) except Exception as err: self._update_main_status("error") - raise err + raise SharedResourceError("Main worker failed during action") from err + # If the action_validator is a callable, use it to validate the result + if callable(self.action_validator) and not self.action_validator(result): + raise SharedResourceError( + f"Action validation failed for {self.action} with {result=}" + ) def wait(self): """Top-level wait function, separating behavior between main and non-main watchers.""" @@ -189,11 +208,16 @@ def __exit__(self, exc_type, exc_value, traceback): raise exc_value if exc_type is None: self.done() + self.unregister() if self.is_main: self._wait_for_status("done") self.resource_file.unlink() else: self._update_status("error") if self.is_main: - self._update_main_status("error") + if self._check_all_status("error"): + # All have failed, delete the file + self.resource_file.unlink() + else: + self._update_main_status("error") raise exc_value diff --git a/tests/new_upgrades/__init__.py b/tests/new_upgrades/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/new_upgrades/conftest.py b/tests/new_upgrades/conftest.py new file mode 100644 index 00000000000..9c00a338756 --- /dev/null +++ b/tests/new_upgrades/conftest.py @@ -0,0 +1,98 @@ +""" +This module is intended to be used for upgrade tests that have a single run stage. +""" +import datetime + +from broker import Broker +import pytest + +from robottelo.config import settings +from robottelo.hosts import Satellite +from robottelo.utils.shared_resource import SharedResource + +pre_upgrade_failed_tests = [] + + +PRE_UPGRADE_TESTS_FILE_OPTION = 'pre_upgrade_tests_file' +PRE_UPGRADE_TESTS_FILE_PATH = '/var/tmp/robottelo_pre_upgrade_failed_tests.json' +PRE_UPGRADE = False +POST_UPGRADE = False +PRE_UPGRADE_MARK = 'pre_upgrade' +POST_UPGRADE_MARK = 'post_upgrade' +TEST_NODE_ID_NAME = '__pytest_node_id' + + +def log(message, level="DEBUG"): + """Pytest has a limitation to use logging.logger from conftest.py + so we need to emulate the logger by std-out the output + """ + now = datetime.datetime.now() + full_message = "{date} - conftest - {level} - {message}\n".format( + date=now.strftime("%Y-%m-%d %H:%M:%S"), level=level, message=message + ) + print(full_message) # noqa + with open('robottelo.log', 'a') as log_file: + log_file.write(full_message) + + +def pytest_configure(config): + """Register custom markers to avoid warnings.""" + markers = [ + "content_upgrades: Upgrade tests that run under .", + ] + for marker in markers: + config.addinivalue_line("markers", marker) + + +def shared_checkout(shared_name): + Satellite(hostname="blank")._swap_nailgun(f"{settings.UPGRADE.FROM_VERSION}.z") + bx_inst = Broker( + workflow=settings.UPGRADE.SATELLITE_DEPLOY_WORKFLOW, + deploy_sat_version=settings.UPGRADE.FROM_VERSION, + host_class=Satellite, + upgrade_group=f"{shared_name}_shared_checkout", + ) + with SharedResource( + resource_name=f"{shared_name}_sat_checkout", + action=bx_inst.checkout, + action_validator=lambda result: isinstance(result, Satellite), + ) as sat_checkout: + sat_checkout.ready() + sat_instance = bx_inst.from_inventory( + filter=f'@inv._broker_args.upgrade_group == "{shared_name}_shared_checkout"' + )[0] + sat_instance.setup() + return sat_instance + + +def shared_checkin(sat_instance): + sat_instance.teardown() + with SharedResource( + resource_name=sat_instance.hostname + "_checkin", + action=Broker(hosts=[sat_instance]).checkin, + ) as sat_checkin: + sat_checkin.ready() + + +@pytest.fixture(scope='session') +def upgrade_action(): + def _upgrade_action(target_sat): + Broker( + job_template=settings.UPGRADE.SATELLITE_UPGRADE_JOB_TEMPLATE, + target_vm=target_sat.name, + sat_version=settings.UPGRADE.TO_VERSION, + tower_inventory=target_sat.tower_inventory, + ).execute() + + return _upgrade_action + + +@pytest.fixture +def content_upgrade_shared_satellite(): + """Mark tests using this fixture with pytest.mark.content_upgrades.""" + sat_instance = shared_checkout("content_upgrade") + with SharedResource( + "content_upgrade_tests", shared_checkin, sat_instance=sat_instance + ) as test_duration: + yield sat_instance + test_duration.ready() diff --git a/tests/new_upgrades/test_contentview.py b/tests/new_upgrades/test_contentview.py new file mode 100644 index 00000000000..c6d0e327909 --- /dev/null +++ b/tests/new_upgrades/test_contentview.py @@ -0,0 +1,124 @@ +"""Test for Content View related Upgrade Scenario's + +:Requirement: UpgradedSatellite + +:CaseAutomation: Automated + +:CaseComponent: ContentViews + +:Team: Phoenix-content + +:CaseImportance: High + +""" +from box import Box +from fauxfactory import gen_alpha +import pytest + +from robottelo.config import settings +from robottelo.constants import RPM_TO_UPLOAD, DataFile +from robottelo.utils.shared_resource import SharedResource + + +@pytest.fixture +def cv_upgrade_setup(content_upgrade_shared_satellite, upgrade_action): + """Pre-upgrade scenario that creates content-view with various repositories. + + :id: preupgrade-a4ebbfa1-106a-4962-9c7c-082833879ae8 + + :steps: + 1. Create custom repositories of yum and file type. + 2. Create content-view. + 3. Add yum and file repositories in the content view. + 4. Publish the content-view. + + :expectedresults: Content-view created with various repositories. + """ + target_sat = content_upgrade_shared_satellite + with SharedResource(target_sat.hostname, upgrade_action, target_sat=target_sat) as sat_upgrade: + test_data = Box( + { + 'target_sat': target_sat, + 'cv': None, + 'org': None, + 'product': None, + 'yum_repo': None, + 'file_repo': None, + } + ) + test_name = f'cv_upgrade_{gen_alpha()}' # unique name for the test + org = target_sat.api.Organization(name=f'{test_name}_org').create() + test_data.org = org + product = target_sat.api.Product(organization=org, name=f'{test_name}_prod').create() + test_data.product = product + yum_repository = target_sat.api.Repository( + product=product, name=f'{test_name}_yum_repo', url=settings.repos.yum_1.url + ).create() + test_data.yum_repo = yum_repository + target_sat.api.Repository.sync(yum_repository) + file_repository = target_sat.api.Repository( + product=product, name=f'{test_name}_file_repo', content_type='file' + ).create() + test_data.file_repo = file_repository + remote_file_path = f'/tmp/{RPM_TO_UPLOAD}' + target_sat.put(DataFile.RPM_TO_UPLOAD, remote_file_path) + file_repository.upload_content(files={'content': DataFile.RPM_TO_UPLOAD.read_bytes()}) + assert 'content' in file_repository.files()['results'][0]['name'] + cv = target_sat.publish_content_view(org, [yum_repository, file_repository]) + assert len(cv.read_json()['versions']) == 1 + sat_upgrade.ready() + yield test_data + + +@pytest.mark.content_upgrades +def test_cv_upgrade_scenario(cv_upgrade_setup): + """After upgrade, the existing content-view(created before upgrade) should be updated. + + :id: postupgrade-a4ebbfa1-106a-4962-9c7c-082833879ae8 + + :steps: + 1. Check yum and file repository which was added in CV before upgrade. + 2. Check the content view which was was created before upgrade. + 3. Remove yum repository from existing CV. + 4. Create new yum repository in existing CV. + 5. Publish content-view + + :expectedresults: After upgrade, + 1. All the repositories should be intact. + 2. Content view created before upgrade should be intact. + 3. The new repository should be added/updated to the CV. + + """ + target_sat = cv_upgrade_setup.target_sat + org = target_sat.api.Organization().search( + query={'search': f'name="{cv_upgrade_setup.org.name}"'} + )[0] + product = target_sat.api.Product(organization=org.id).search( + query={'search': f'name="{cv_upgrade_setup.product.name}"'} + )[0] + cv = target_sat.api.ContentView(organization=org.id).search( + query={'search': f'name="{cv_upgrade_setup.cv.name}"'} + )[0] + target_sat.api.Repository(organization=org.id).search( + query={'search': f'name="{cv_upgrade_setup.yum_repo.name}"'} + )[0] + target_sat.api.Repository(organization=org.id).search( + query={'search': f'name="{cv_upgrade_setup.file_repo.name}"'} + )[0] + cv.repository = [] + cv.update(['repository']) + assert len(cv.read_json()['repositories']) == 0 + + yum_repository2 = target_sat.api.Repository( + product=product, name='cv_upgrade_yum_repos2', url=settings.repos.yum_2.url + ).create() + yum_repository2.sync() + cv.repository = [yum_repository2] + cv.update(['repository']) + assert cv.read_json()['repositories'][0]['name'] == yum_repository2.name + + cv.publish() + assert len(cv.read_json()['versions']) == 2 + content_view_json = cv.read_json()['environments'][0] + cv.delete_from_environment(content_view_json['id']) + assert len(cv.read_json()['environments']) == 0 diff --git a/tests/new_upgrades/test_repository.py b/tests/new_upgrades/test_repository.py new file mode 100644 index 00000000000..a81746932bf --- /dev/null +++ b/tests/new_upgrades/test_repository.py @@ -0,0 +1,117 @@ +"""Test for Repository related Upgrade Scenarios + +:Requirement: UpgradedSatellite + +:CaseAutomation: Automated + +:CaseComponent: Repositories + +:Team: Phoenix-content + +:CaseImportance: High + +""" +from box import Box +import pytest + +from robottelo.config import settings +from robottelo.constants import ( + FAKE_0_CUSTOM_PACKAGE_NAME, + FAKE_4_CUSTOM_PACKAGE_NAME, +) +from robottelo.hosts import ContentHost +from robottelo.utils.shared_resource import SharedResource + + +@pytest.fixture +def custom_repo_check_setup(sat_upgrade_chost, content_upgrade_shared_satellite, upgrade_action): + """This is pre-upgrade scenario test to verify if we can create a + custom repository and consume it via content host. + + :id: preupgrade-eb6831b1-c5b6-4941-a325-994a09467478 + + :steps: + 1. Before Satellite upgrade. + 2. Create new Organization, Location. + 3. Create Product, custom repo, cv. + 4. Create activation key and add subscription. + 5. Create a content host, register and install package on it. + + :expectedresults: + + 1. Custom repo is created. + 2. Package is installed on Content host. + + """ + target_sat = content_upgrade_shared_satellite + with SharedResource(target_sat.hostname, upgrade_action, target_sat=target_sat) as sat_upgrade: + test_data = Box( + { + 'target_sat': target_sat, + 'rhel_client': sat_upgrade_chost, + 'lce': None, + 'repo': None, + 'content_view': None, + } + ) + org = target_sat.api.Organization().create() + lce = target_sat.api.LifecycleEnvironment(organization=org).create() + test_data.lce = lce + product = target_sat.api.Product(organization=org).create() + repo = target_sat.api.Repository(product=product.id, url=settings.repos.yum_1.url).create() + test_data.repo = repo + repo.sync() + content_view = target_sat.publish_content_view(org, repo) + test_data.content_view = content_view + content_view.version[0].promote(data={'environment_ids': lce.id}) + ak = target_sat.api.ActivationKey( + content_view=content_view, organization=org.id, environment=lce + ).create() + if not target_sat.is_sca_mode_enabled(org.id): + subscription = target_sat.api.Subscription(organization=org).search( + query={'search': f'name={product.name}'} + )[0] + ak.add_subscriptions(data={'subscription_id': subscription.id}) + sat_upgrade_chost.install_katello_ca(target_sat) + sat_upgrade_chost.register_contenthost(org.label, ak.name) + sat_upgrade_chost.execute('subscription-manager repos --enable=*;yum clean all') + result = sat_upgrade_chost.execute(f'yum install -y {FAKE_0_CUSTOM_PACKAGE_NAME}') + assert result.status == 0 + sat_upgrade.ready() + yield test_data + + +@pytest.mark.content_upgrades +def test_scenario_custom_repo_check(custom_repo_check_setup): + """This is post-upgrade scenario test to verify if we can alter the + created custom repository and satellite will be able to sync back + the repo. + + :id: postupgrade-5c793577-e573-46a7-abbf-b6fd1f20b06e + + :steps: + 1. Remove old and add new package into custom repo. + 2. Sync repo , publish the new version of cv. + 3. Try to install new package on client. + + + :expectedresults: Content host should be able to pull the new rpm. + + """ + test_data = custom_repo_check_setup + target_sat = test_data.target_sat + repo = target_sat.api.Repository(name=test_data.repo.name).search()[0] + repo.sync() + + content_view = target_sat.api.ContentView(name=test_data.content_view.name).search()[0] + content_view.publish() + + content_view = target_sat.api.ContentView(name=test_data.content_view.name).search()[0] + latest_cvv_id = sorted(cvv.id for cvv in content_view.version)[-1] + target_sat.api.ContentViewVersion(id=latest_cvv_id).promote( + data={'environment_ids': test_data.lce.id} + ) + + rhel_client = ContentHost.get_host_by_hostname(test_data.rhel_client.hostname) + result = rhel_client.execute(f'yum install -y {FAKE_4_CUSTOM_PACKAGE_NAME}') + assert result.status == 0 diff --git a/tests/robottelo/test_shared_resource.py b/tests/robottelo/test_shared_resource.py index ff2146cd80a..d0745bc1fce 100644 --- a/tests/robottelo/test_shared_resource.py +++ b/tests/robottelo/test_shared_resource.py @@ -11,6 +11,7 @@ def upgrade_action(*args, **kwargs): print(f"Upgrading satellite with {args=} and {kwargs=}") time.sleep(1) print("Satellite upgraded!") + return True def run_resource(resource_name): @@ -24,7 +25,14 @@ def run_resource(resource_name): def test_shared_resource(): """Test the SharedResource class.""" - with SharedResource("test_resource", upgrade_action, 1, 2, 3, foo="bar") as resource: + + def action_validator(result): + return result is True + + shared_args = (1, 2, 3) + with SharedResource( + "test_resource", upgrade_action, *shared_args, action_validator=action_validator, foo="bar" + ) as resource: assert Path("/tmp/test_resource.shared").exists() assert resource.is_main assert not resource.is_recovering