[POC] New-Style Upgrade Tests

SharedResource:
- Added the ability to validate the result of a given action function via an action_validator function.
- Made an improvement to exiting under error conditions that improved
  tracking file cleanup.

New directory for new-style upgrades located at tests/new_upgrades. This
will help to keep changes isolated from the existing upgrade tests.

new_upgrades/conftest:
- Removed the requirement for all upgrade tests to be marked as pre/post
- Introduced fixtures that coordinate checkout/checkin actions between
  multiple xdist workers.
- Introduced a fixture that performs an upgrade on a target satellite
- Introduced a fixture that is used for two test conversions in
  different modules.

test conversions:
- test_cv_upgrade_scenario and test_scenario_custom_repo_check converted
- pre-upgrade tests are now pre-upgrade fixtures that perform setup and
  yield their data in Box objects instead of saving to disk
- post-upgrade tests can now directly access the setup objects by
  inheriting the pre-upgrade fixture results

settings:
- Added SATELLITE_DEPLOY_WORKFLOW and SATELLITE_UPGRADE_JOB_TEMPLATE to
  upgrade.yaml

conf/upgrade.yaml.template

@@ -5,6 +5,10 @@ UPGRADE:
   TO_VERSION: "6.9"
   # Satellite, Capsule hosts RHEL operating system version.
   OS: "rhel7"
+  # The workflow Broker should use to checkout a to-be-upgraded Satellite
+  SATELLITE_DEPLOY_WORKFLOW: deploy-satellite-upgrade
+  # The job template Broker should use to upgrade a Satellite
+  SATELLITE_UPGRADE_JOB_TEMPLATE: satellite-upgrade
   # Capsule's activation key will only be available when we spawn the VM using upgrade template.
   CAPSULE_AK:
     RHEL6:

requirements.txt

@@ -1,16 +1,14 @@
 # Version updates managed by dependabot
 
 betelgeuse==1.11.0
-# broker[docker]==0.4.1 - Temporarily disabled, see below
+broker[docker]==0.4.8
 cryptography==42.0.5
 deepdiff==6.7.1
-docker==7.0.0  # Temporary until Broker is back on PyPi
 dynaconf[vault]==3.2.4
 fauxfactory==3.1.0
 jinja2==3.1.3
 manifester==0.0.14
 navmazing==1.2.2
-paramiko==3.4.0  # Temporary until Broker is back on PyPi
 productmd==1.38
 pyotp==2.9.0
 python-box==7.1.1
@@ -30,11 +28,6 @@ wait-for==1.2.0
 wrapanapi==3.6.0
 
 # Get airgun, nailgun and upgrade from master
-git+https://github.com/SatelliteQE/airgun.git@master#egg=airgun
-git+https://github.com/SatelliteQE/nailgun.git@master#egg=nailgun
-# Broker currently is unable to push to PyPi due to [1] and [2]
-# In the meantime, we install directly from the repo
-# [1] - https://github.com/ParallelSSH/ssh2-python/issues/193
-# [2] - https://github.com/pypi/warehouse/issues/7136
-git+https://github.com/SatelliteQE/[email protected]#egg=broker
+airgun @ git+https://github.com/SatelliteQE/airgun.git@master#egg=airgun
+nailgun @ git+https://github.com/SatelliteQE/nailgun.git@master#egg=nailgun
 --editable .

robottelo/utils/shared_resource.py

@@ -29,6 +29,10 @@
 from broker.helpers import FileLock
 
 
+class SharedResourceError(Exception):
+    """An exception class for SharedResource errors."""
+
+
 class SharedResource:
     """A class representing a shared resource.
 
@@ -43,19 +47,21 @@ class SharedResource:
         is_recovering (bool): Whether the current instance is recovering from an error or not.
     """
 
-    def __init__(self, resource_name, action, *action_args, **action_kwargs):
+    def __init__(self, resource_name, action, *action_args, action_validator=None, **action_kwargs):
         """Initializes a new instance of the SharedResource class.
 
         Args:
             resource_name (str): The name of the shared resource.
             action (function): The function to be executed when the resource is ready.
             action_args (tuple): The arguments to be passed to the action function.
+            action_validator (function): The function to validate the action results.
             action_kwargs (dict): The keyword arguments to be passed to the action function.
         """
         self.resource_file = Path(f"/tmp/{resource_name}.shared")
         self.lock_file = FileLock(self.resource_file)
         self.id = str(uuid4().fields[-1])
         self.action = action
+        self.action_validator = action_validator
         self.action_is_recoverable = action_kwargs.pop("action_is_recoverable", False)
         self.action_args = action_args
         self.action_kwargs = action_kwargs
@@ -151,6 +157,14 @@ def register(self):
             curr_data["statuses"][self.id] = "pending"
             self.resource_file.write_text(json.dumps(curr_data, indent=4))
 
+    def unregister(self):
+        """Unregisters the current process as a watcher."""
+        with self.lock_file:
+            curr_data = json.loads(self.resource_file.read_text())
+            curr_data["watchers"].remove(self.id)
+            del curr_data["statuses"][self.id]
+            self.resource_file.write_text(json.dumps(curr_data, indent=4))
+
     def ready(self):
         """Marks the current process as ready to perform the action."""
         self._update_status("ready")
@@ -163,10 +177,15 @@ def done(self):
     def act(self):
         """Attempt to perform the action."""
         try:
-            self.action(*self.action_args, **self.action_kwargs)
+            result = self.action(*self.action_args, **self.action_kwargs)
         except Exception as err:
             self._update_main_status("error")
-            raise err
+            raise SharedResourceError("Main worker failed during action") from err
+        # If the action_validator is a callable, use it to validate the result
+        if callable(self.action_validator) and not self.action_validator(result):
+            raise SharedResourceError(
+                f"Action validation failed for {self.action} with {result=}"
+            )
 
     def wait(self):
         """Top-level wait function, separating behavior between main and non-main watchers."""
@@ -189,11 +208,16 @@ def __exit__(self, exc_type, exc_value, traceback):
             raise exc_value
         if exc_type is None:
             self.done()
+            self.unregister()
             if self.is_main:
                 self._wait_for_status("done")
                 self.resource_file.unlink()
         else:
             self._update_status("error")
             if self.is_main:
-                self._update_main_status("error")
+                if self._check_all_status("error"):
+                    # All have failed, delete the file
+                    self.resource_file.unlink()
+                else:
+                    self._update_main_status("error")
             raise exc_value

tests/new_upgrades/conftest.py

@@ -0,0 +1,98 @@
+"""
+This module is intended to be used for upgrade tests that have a single run stage.
+"""
+import datetime
+
+from broker import Broker
+import pytest
+
+from robottelo.config import settings
+from robottelo.hosts import Satellite
+from robottelo.utils.shared_resource import SharedResource
+
+pre_upgrade_failed_tests = []
+
+
+PRE_UPGRADE_TESTS_FILE_OPTION = 'pre_upgrade_tests_file'
+PRE_UPGRADE_TESTS_FILE_PATH = '/var/tmp/robottelo_pre_upgrade_failed_tests.json'
+PRE_UPGRADE = False
+POST_UPGRADE = False
+PRE_UPGRADE_MARK = 'pre_upgrade'
+POST_UPGRADE_MARK = 'post_upgrade'
+TEST_NODE_ID_NAME = '__pytest_node_id'
+
+
+def log(message, level="DEBUG"):
+    """Pytest has a limitation to use logging.logger from conftest.py
+    so we need to emulate the logger by std-out the output
+    """
+    now = datetime.datetime.now()
+    full_message = "{date} - conftest - {level} - {message}\n".format(
+        date=now.strftime("%Y-%m-%d %H:%M:%S"), level=level, message=message
+    )
+    print(full_message)  # noqa
+    with open('robottelo.log', 'a') as log_file:
+        log_file.write(full_message)
+
+
+def pytest_configure(config):
+    """Register custom markers to avoid warnings."""
+    markers = [
+        "content_upgrades: Upgrade tests that run under .",
+    ]
+    for marker in markers:
+        config.addinivalue_line("markers", marker)
+
+
+def shared_checkout(shared_name):
+    Satellite(hostname="blank")._swap_nailgun(f"{settings.UPGRADE.FROM_VERSION}.z")
+    bx_inst = Broker(
+        workflow=settings.UPGRADE.SATELLITE_DEPLOY_WORKFLOW,
+        deploy_sat_version=settings.UPGRADE.FROM_VERSION,
+        host_class=Satellite,
+        upgrade_group=f"{shared_name}_shared_checkout",
+    )
+    with SharedResource(
+        resource_name=f"{shared_name}_sat_checkout",
+        action=bx_inst.checkout,
+        action_validator=lambda result: isinstance(result, Satellite),
+    ) as sat_checkout:
+        sat_checkout.ready()
+        sat_instance = bx_inst.from_inventory(
+            filter=f'@inv._broker_args.upgrade_group == "{shared_name}_shared_checkout"'
+        )[0]
+        sat_instance.setup()
+    return sat_instance
+
+
+def shared_checkin(sat_instance):
+    sat_instance.teardown()
+    with SharedResource(
+        resource_name=sat_instance.hostname + "_checkin",
+        action=Broker(hosts=[sat_instance]).checkin,
+    ) as sat_checkin:
+        sat_checkin.ready()
+
+
+@pytest.fixture(scope='session')
+def upgrade_action():
+    def _upgrade_action(target_sat):
+        Broker(
+            job_template=settings.UPGRADE.SATELLITE_UPGRADE_JOB_TEMPLATE,
+            target_vm=target_sat.name,
+            sat_version=settings.UPGRADE.TO_VERSION,
+            tower_inventory=target_sat.tower_inventory,
+        ).execute()
+
+    return _upgrade_action
+
+
+@pytest.fixture
+def content_upgrade_shared_satellite():
+    """Mark tests using this fixture with pytest.mark.content_upgrades."""
+    sat_instance = shared_checkout("content_upgrade")
+    with SharedResource(
+        "content_upgrade_tests", shared_checkin, sat_instance=sat_instance
+    ) as test_duration:
+        yield sat_instance
+        test_duration.ready()

tests/new_upgrades/test_contentview.py

@@ -0,0 +1,124 @@
+"""Test for Content View related Upgrade Scenario's
+
+:Requirement: UpgradedSatellite
+
+:CaseAutomation: Automated
+
+:CaseComponent: ContentViews
+
+:Team: Phoenix-content
+
+:CaseImportance: High
+
+"""
+from box import Box
+from fauxfactory import gen_alpha
+import pytest
+
+from robottelo.config import settings
+from robottelo.constants import RPM_TO_UPLOAD, DataFile
+from robottelo.utils.shared_resource import SharedResource
+
+
+@pytest.fixture
+def cv_upgrade_setup(content_upgrade_shared_satellite, upgrade_action):
+    """Pre-upgrade scenario that creates content-view with various repositories.
+
+    :id: preupgrade-a4ebbfa1-106a-4962-9c7c-082833879ae8
+
+    :steps:
+        1. Create custom repositories of yum and file type.
+        2. Create content-view.
+        3. Add yum and file repositories in the content view.
+        4. Publish the content-view.
+
+    :expectedresults: Content-view created with various repositories.
+    """
+    target_sat = content_upgrade_shared_satellite
+    with SharedResource(target_sat.hostname, upgrade_action, target_sat=target_sat) as sat_upgrade:
+        test_data = Box(
+            {
+                'target_sat': target_sat,
+                'cv': None,
+                'org': None,
+                'product': None,
+                'yum_repo': None,
+                'file_repo': None,
+            }
+        )
+        test_name = f'cv_upgrade_{gen_alpha()}'  # unique name for the test
+        org = target_sat.api.Organization(name=f'{test_name}_org').create()
+        test_data.org = org
+        product = target_sat.api.Product(organization=org, name=f'{test_name}_prod').create()
+        test_data.product = product
+        yum_repository = target_sat.api.Repository(
+            product=product, name=f'{test_name}_yum_repo', url=settings.repos.yum_1.url
+        ).create()
+        test_data.yum_repo = yum_repository
+        target_sat.api.Repository.sync(yum_repository)
+        file_repository = target_sat.api.Repository(
+            product=product, name=f'{test_name}_file_repo', content_type='file'
+        ).create()
+        test_data.file_repo = file_repository
+        remote_file_path = f'/tmp/{RPM_TO_UPLOAD}'
+        target_sat.put(DataFile.RPM_TO_UPLOAD, remote_file_path)
+        file_repository.upload_content(files={'content': DataFile.RPM_TO_UPLOAD.read_bytes()})
+        assert 'content' in file_repository.files()['results'][0]['name']
+        cv = target_sat.publish_content_view(org, [yum_repository, file_repository])
+        assert len(cv.read_json()['versions']) == 1
+        sat_upgrade.ready()
+        yield test_data
+
+
+@pytest.mark.content_upgrades
+def test_cv_upgrade_scenario(cv_upgrade_setup):
+    """After upgrade, the existing content-view(created before upgrade) should be updated.
+
+    :id: postupgrade-a4ebbfa1-106a-4962-9c7c-082833879ae8
+
+    :steps:
+        1. Check yum and file repository which was added in CV before upgrade.
+        2. Check the content view which was was created before upgrade.
+        3. Remove yum repository from existing CV.
+        4. Create new yum repository in existing CV.
+        5. Publish content-view
+
+    :expectedresults: After upgrade,
+        1. All the repositories should be intact.
+        2. Content view created before upgrade should be intact.
+        3. The new repository should be added/updated to the CV.
+
+    """
+    target_sat = cv_upgrade_setup.target_sat
+    org = target_sat.api.Organization().search(
+        query={'search': f'name="{cv_upgrade_setup.org.name}"'}
+    )[0]
+    product = target_sat.api.Product(organization=org.id).search(
+        query={'search': f'name="{cv_upgrade_setup.product.name}"'}
+    )[0]
+    cv = target_sat.api.ContentView(organization=org.id).search(
+        query={'search': f'name="{cv_upgrade_setup.cv.name}"'}
+    )[0]
+    target_sat.api.Repository(organization=org.id).search(
+        query={'search': f'name="{cv_upgrade_setup.yum_repo.name}"'}
+    )[0]
+    target_sat.api.Repository(organization=org.id).search(
+        query={'search': f'name="{cv_upgrade_setup.file_repo.name}"'}
+    )[0]
+    cv.repository = []
+    cv.update(['repository'])
+    assert len(cv.read_json()['repositories']) == 0
+
+    yum_repository2 = target_sat.api.Repository(
+        product=product, name='cv_upgrade_yum_repos2', url=settings.repos.yum_2.url
+    ).create()
+    yum_repository2.sync()
+    cv.repository = [yum_repository2]
+    cv.update(['repository'])
+    assert cv.read_json()['repositories'][0]['name'] == yum_repository2.name
+
+    cv.publish()
+    assert len(cv.read_json()['versions']) == 2
+    content_view_json = cv.read_json()['environments'][0]
+    cv.delete_from_environment(content_view_json['id'])
+    assert len(cv.read_json()['environments']) == 0