diff --git a/.gitignore b/.gitignore index 07bd9ff5076..37a548e7d4f 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,7 @@ ven*/ # pytest-fixture-tools artifacts artifacts/ + +# pyvcloud artifacts +# workaround till https://github.com/vmware/pyvcloud/pull/766 is merged and released +**vcd_sdk.log diff --git a/pytest_fixtures/core/broker.py b/pytest_fixtures/core/broker.py index 36822ad2461..4c89dea35c5 100644 --- a/pytest_fixtures/core/broker.py +++ b/pytest_fixtures/core/broker.py @@ -5,6 +5,7 @@ from broker import Broker from robottelo.config import settings +from robottelo.hosts import ContentHostError from robottelo.hosts import lru_sat_ready_rhel from robottelo.hosts import Satellite @@ -13,12 +14,9 @@ def _default_sat(align_to_satellite): """Returns a Satellite object for settings.server.hostname""" if settings.server.hostname: - hosts = Broker(host_class=Satellite).from_inventory( - filter=f'@inv.hostname == "{settings.server.hostname}"' - ) - if hosts: - return hosts[0] - else: + try: + return Satellite.get_host_by_hostname(settings.server.hostname) + except ContentHostError: return Satellite() diff --git a/pytest_fixtures/core/sat_cap_factory.py b/pytest_fixtures/core/sat_cap_factory.py index 0ca8431ff9a..678fdde2e0c 100644 --- a/pytest_fixtures/core/sat_cap_factory.py +++ b/pytest_fixtures/core/sat_cap_factory.py @@ -331,7 +331,5 @@ def installer_satellite(request): if 'sanity' not in request.config.option.markexpr: sanity_sat = Satellite(sat.hostname) sanity_sat.unregister() - broker_sat = Broker(host_class=Satellite).from_inventory( - filter=f'@inv.hostname == "{sanity_sat.hostname}"' - )[0] + broker_sat = Satellite.get_host_by_hostname(sanity_sat.hostname) Broker(hosts=[broker_sat]).checkin() diff --git a/pytest_fixtures/core/upgrade.py b/pytest_fixtures/core/upgrade.py index fd9c2fd364b..291bae18fbf 100644 --- a/pytest_fixtures/core/upgrade.py +++ b/pytest_fixtures/core/upgrade.py @@ -38,6 +38,6 @@ def pre_configured_capsule(worker_id, session_target_sat): logger.debug(f'Capsules found: {intersect}') assert len(intersect) == 1, "More than one Capsule found in the inventory" target_capsule = intersect.pop() - hosts = Broker(host_class=Capsule).from_inventory(filter=f'@inv.hostname == "{target_capsule}"') + host = Capsule.get_host_by_hostname(target_capsule) logger.info(f'xdist worker {worker_id} was assigned pre-configured Capsule {target_capsule}') - return hosts[0] + return host diff --git a/pytest_fixtures/core/xdist.py b/pytest_fixtures/core/xdist.py index 7c4917725be..18088f8a572 100644 --- a/pytest_fixtures/core/xdist.py +++ b/pytest_fixtures/core/xdist.py @@ -19,9 +19,7 @@ def align_to_satellite(request, worker_id, satellite_factory): if settings.server.hostname: sanity_sat = Satellite(settings.server.hostname) sanity_sat.unregister() - broker_sat = Broker(host_class=Satellite).from_inventory( - filter=f'@inv.hostname == "{sanity_sat.hostname}"' - )[0] + broker_sat = Satellite.get_host_by_hostname(sanity_sat.hostname) Broker(hosts=[broker_sat]).checkin() else: # clear any hostname that may have been previously set @@ -35,9 +33,7 @@ def align_to_satellite(request, worker_id, satellite_factory): # attempt to add potential satellites from the broker inventory file if settings.server.inventory_filter: - hosts = Broker(host_class=Satellite).from_inventory( - filter=settings.server.inventory_filter - ) + hosts = Satellite.get_hosts_from_inventory(filter=settings.server.inventory_filter) settings.server.hostnames += [host.hostname for host in hosts] # attempt to align a worker to a satellite diff --git a/robottelo/host_helpers/contenthost_mixins.py b/robottelo/host_helpers/contenthost_mixins.py index a4ab9457762..30fbf576e1f 100644 --- a/robottelo/host_helpers/contenthost_mixins.py +++ b/robottelo/host_helpers/contenthost_mixins.py @@ -20,10 +20,21 @@ def _v_major(self): @cached_property def REPOSET(self): - return { - 'rhel': constants.REPOSET[f'rhel{self._v_major}'], - 'rhst': constants.REPOSET[f'rhst{self._v_major}'], - } + try: + if self._v_major > 7: + sys_reposets = { + 'rhel_bos': constants.REPOSET[f'rhel{self._v_major}_bos'], + 'rhel_aps': constants.REPOSET[f'rhel{self._v_major}_aps'], + } + else: + sys_reposets = { + 'rhel': constants.REPOSET[f'rhel{self._v_major}'], + 'rhscl': constants.REPOSET[f'rhscl{self._v_major}'], + } + reposets = {'rhst': constants.REPOSET[f'rhst{self._v_major}']} + except KeyError as err: + raise ValueError(f'Unsupported system version: {self._v_major}') from err + return sys_reposets | reposets @cached_property def REPOS(self): diff --git a/robottelo/hosts.py b/robottelo/hosts.py index 0eeb5b58c2c..7f30dc4d335 100644 --- a/robottelo/hosts.py +++ b/robottelo/hosts.py @@ -1,3 +1,4 @@ +import contextlib import importlib import io import json @@ -198,6 +199,26 @@ def __init__(self, hostname, auth=None, **kwargs): self.blank = kwargs.get('blank', False) super().__init__(hostname=hostname, **kwargs) + @classmethod + def get_hosts_from_inventory(cls, filter): + """Get an instance of a host from inventory using a filter""" + inv_hosts = Broker(host_class=cls).from_inventory(filter) + logger.debug('Found %s instances from inventory by filter: %s', len(inv_hosts), filter) + return inv_hosts + + @classmethod + def get_host_by_hostname(cls, hostname): + """Get an instance of a host from inventory by hostname""" + logger.info('Getting %s instance from inventory by hostname: %s', cls.__name__, hostname) + inv_hosts = cls.get_hosts_from_inventory(filter=f'@inv.hostname == "{hostname}"') + if not inv_hosts: + raise ContentHostError(f'No {cls.__name__} found in inventory by hostname {hostname}') + if len(inv_hosts) > 1: + raise ContentHostError( + f'Multiple {cls.__name__} found in inventory by hostname {hostname}' + ) + return inv_hosts[0] + @property def satellite(self): if not self._satellite: @@ -248,31 +269,107 @@ def arch(self): @cached_property def _redhat_release(self): - """Process redhat-release file for distro and version information""" + """Process redhat-release file for distro and version information + This is a fallback for when /etc/os-release is not available + """ result = self.execute('cat /etc/redhat-release') if result.status != 0: raise ContentHostError(f'Not able to cat /etc/redhat-release "{result.stderr}"') - match = re.match(r'(?P.+) release (?P\d+)(.(?P\d+))?', result.stdout) + match = re.match(r'(?P.+) release (?P\d+)(.(?P\d+))?', result.stdout) if match is None: raise ContentHostError(f'Not able to parse release string "{result.stdout}"') - return match.groupdict() + r_release = match.groupdict() + + # /etc/os-release compatibility layer + r_release['VERSION_ID'] = r_release['major'] + # not every release have a minor version + r_release['VERSION_ID'] += f'.{r_release["minor"]}' if r_release['minor'] else '' + + distro_map = { + 'Fedora': {'NAME': 'Fedora Linux', 'ID': 'fedora'}, + 'CentOS': {'ID': 'centos'}, + 'Red Hat Enterprise Linux': {'ID': 'rhel'}, + } + # Use the version map to set the NAME and ID fields + for distro, properties in distro_map.items(): + if distro in r_release['NAME']: + r_release.update(properties) + break + return r_release @cached_property + def _os_release(self): + """Process os-release file for distro and version information""" + facts = {} + regex = r'^(["\'])(.*)(\1)$' + result = self.execute('cat /etc/os-release') + if result.status != 0: + logger.info( + f'Not able to cat /etc/os-release "{result.stderr}", ' + 'falling back to /etc/redhat-release' + ) + return self._redhat_release + for ln in [line for line in result.stdout.splitlines() if line.strip()]: + line = ln.strip() + if line.startswith('#'): + continue + key, value = line.split('=') + if key and value: + facts[key] = re.sub(regex, r'\2', value).replace('\\', '') + return facts + + @property def os_distro(self): """Get host's distro information""" - groups = self._redhat_release - return groups['distro'] + return self._os_release['NAME'] - @cached_property + @property def os_version(self): """Get host's OS version information :returns: A ``packaging.version.Version`` instance """ - groups = self._redhat_release - minor_version = '' if groups['minor'] is None else f'.{groups["minor"]}' - version_string = f'{groups["major"]}{minor_version}' - return Version(version=version_string) + return Version(self._os_release['VERSION_ID']) + + @property + def os_id(self): + """Get host's OS ID information""" + return self._os_release['ID'] + + @cached_property + def is_el(self): + """Boolean representation of whether this host is an EL host""" + return self.execute('stat /etc/redhat-release').status == 0 + + @property + def is_rhel(self): + """Boolean representation of whether this host is a RHEL host""" + return self.os_id == 'rhel' + + @property + def is_centos(self): + """Boolean representation of whether this host is a CentOS host""" + return self.os_id == 'centos' + + def list_cached_properties(self): + """Return a list of cached property names of this class""" + import inspect + + return [ + name + for name, value in inspect.getmembers(self.__class__) + if isinstance(value, cached_property) + ] + + def get_cached_properties(self): + """Return a dictionary of cached properties for this class""" + return {name: getattr(self, name) for name in self.list_cached_properties()} + + def clean_cached_properties(self): + """Delete all cached properties for this class""" + for name in self.list_cached_properties(): + with contextlib.suppress(KeyError): # ignore if property is not cached + del self.__dict__[name] def setup(self): if not self.blank: @@ -280,18 +377,9 @@ def setup(self): def teardown(self): if not self.blank and not getattr(self, '_skip_context_checkin', False): - if self.nailgun_host: - self.nailgun_host.delete() self.unregister() - # Strip most unnecessary attributes from our instance for checkin - keep_keys = set(self.to_dict()) | { - 'release', - '_prov_inst', - '_cont_inst', - '_skip_context_checkin', - } - self.__dict__ = {k: v for k, v in self.__dict__.items() if k in keep_keys} - self.__class__ = Host + if type(self) is not Satellite and self.nailgun_host: + self.nailgun_host.delete() def power_control(self, state=VmState.RUNNING, ensure=True): """Lookup the host workflow for power on and execute @@ -305,6 +393,8 @@ def power_control(self, state=VmState.RUNNING, ensure=True): BrokerError: various error types to do with broker execution ContentHostError: if the workflow status isn't successful and broker didn't raise """ + if getattr(self, '_cont_inst', None): + raise NotImplementedError('Power control not supported for container instances') try: vm_operation = POWER_OPERATIONS.get(state) workflow_name = settings.broker.host_workflows.power_control @@ -331,8 +421,23 @@ def power_control(self, state=VmState.RUNNING, ensure=True): self.connect, fail_condition=lambda res: res is not None, handle_exception=True ) # really broad diaper here, but connection exceptions could be a ton of types - except TimedOutError: - raise ContentHostError('Unable to connect to host that should be running') + except TimedOutError as toe: + raise ContentHostError('Unable to connect to host that should be running') from toe + + def wait_for_connection(self, timeout=180): + try: + wait_for( + self.connect, + fail_condition=lambda res: res is not None, + handle_exception=True, + raise_original=True, + timeout=timeout, + delay=1, + ) + except (ConnectionRefusedError, ConnectionAbortedError, TimedOutError) as err: + raise ContentHostError( + f'Unable to establsh SSH connection to host {self} after {timeout} seconds' + ) from err def download_file(self, file_url, local_path=None, file_name=None): """Downloads file from given fileurl to directory specified by local_path by given filename @@ -546,6 +651,8 @@ def remove_katello_ca(self): :return: None. :raises robottelo.hosts.ContentHostError: If katello-ca wasn't removed. """ + # unregister host from CDN to avoid subscription leakage + self.execute('subscription-manager unregister') # Not checking the status here, as rpm can be not even installed # and deleting may fail self.execute('yum erase -y $(rpm -qa |grep katello-ca-consumer)') @@ -1405,12 +1512,9 @@ def satellite(self): answers = Box(yaml.load(data, yaml.FullLoader)) sat_hostname = urlparse(answers.foreman_proxy.foreman_base_url).netloc # get the Satellite hostname from the answer file - hosts = Broker(host_class=Satellite).from_inventory( - filter=f'@inv.hostname == "{sat_hostname}"' - ) - if hosts: - self._satellite = hosts[0] - else: + try: + self._satellite = Satellite.get_host_by_hostname(sat_hostname) + except ContentHostError: logger.debug( f'No Satellite host found in inventory for {self.hostname}. ' 'Satellite object with the same hostname will be created anyway.' diff --git a/tests/foreman/api/test_provisioning.py b/tests/foreman/api/test_provisioning.py index 4d54a403672..30fd3587c66 100644 --- a/tests/foreman/api/test_provisioning.py +++ b/tests/foreman/api/test_provisioning.py @@ -104,17 +104,7 @@ def test_rhel_pxe_provisioning( provisioning_host.blank = False # Wait for the host to be rebooted and SSH daemon to be started. - try: - wait_for( - provisioning_host.connect, - fail_condition=lambda res: res is not None, - handle_exception=True, - raise_original=True, - timeout=180, - delay=1, - ) - except ConnectionRefusedError: - raise ConnectionRefusedError("Timed out waiting for SSH daemon to start on the host") + provisioning_host.wait_for_connection() # Perform version check host_os = host.operatingsystem.read() diff --git a/tests/foreman/api/test_provisioning_puppet.py b/tests/foreman/api/test_provisioning_puppet.py index 2a317c071be..195f458b3b9 100644 --- a/tests/foreman/api/test_provisioning_puppet.py +++ b/tests/foreman/api/test_provisioning_puppet.py @@ -185,17 +185,7 @@ def test_host_provisioning_with_external_puppetserver( provisioning_host.blank = False # Wait for the host to be rebooted and SSH daemon to be started. - try: - wait_for( - provisioning_host.connect, - fail_condition=lambda res: res is not None, - handle_exception=True, - raise_original=True, - timeout=180, - delay=1, - ) - except ConnectionRefusedError: - raise ConnectionRefusedError('Timed out waiting for SSH daemon to start on the host') + provisioning_host.wait_for_connection() # Perform version check host_os = host.operatingsystem.read() diff --git a/tests/foreman/destructive/test_leapp_satellite.py b/tests/foreman/destructive/test_leapp_satellite.py index e4683e2f067..5bd6a19b9d7 100644 --- a/tests/foreman/destructive/test_leapp_satellite.py +++ b/tests/foreman/destructive/test_leapp_satellite.py @@ -47,10 +47,12 @@ def test_positive_leapp(target_sat): ) # Recreate the session object within a Satellite object after upgrading target_sat.connect() + # Clean cached properties after the upgrade + target_sat.clean_cached_properties() # Get RHEL version after upgrading - result = target_sat.execute('cat /etc/redhat-release | grep -Po "\\d"') + res_rhel_version = target_sat.os_version.major # Check if RHEL was upgraded - assert result.stdout[0] == str(orig_rhel_ver + 1), 'RHEL was not upgraded' + assert res_rhel_version == orig_rhel_ver + 1, 'RHEL was not upgraded' # Check satellite's health sat_health = target_sat.execute('satellite-maintain health check') assert sat_health.status == 0, 'Satellite health check failed' diff --git a/tests/upgrades/test_client.py b/tests/upgrades/test_client.py index 95c93aebcae..ed2069af5e3 100644 --- a/tests/upgrades/test_client.py +++ b/tests/upgrades/test_client.py @@ -20,7 +20,6 @@ :Upstream: No """ import pytest -from broker import Broker from robottelo.constants import FAKE_0_CUSTOM_PACKAGE_NAME from robottelo.constants import FAKE_4_CUSTOM_PACKAGE_NAME @@ -93,17 +92,12 @@ def test_post_scenario_pre_client_package_installation( :expectedresults: The package is installed on client """ - client_name = pre_upgrade_data.get('rhel_client') - client_id = ( - module_target_sat.api.Host().search(query={'search': f'name={client_name}'})[0].id - ) + client_hostname = pre_upgrade_data.get('rhel_client') + rhel_client = ContentHost.get_host_by_hostname(client_hostname) module_target_sat.cli.Host.package_install( - {'host-id': client_id, 'packages': FAKE_0_CUSTOM_PACKAGE_NAME} + {'host-id': rhel_client.nailgun_host.id, 'packages': FAKE_0_CUSTOM_PACKAGE_NAME} ) - # Verifies that package is really installed - rhel_client = Broker(host_class=ContentHost).from_inventory( - filter=f'@inv.hostname == "{client_name}"' - )[0] + # Verify that package is really installed result = rhel_client.execute(f"rpm -q {FAKE_0_CUSTOM_PACKAGE_NAME}") assert FAKE_0_CUSTOM_PACKAGE_NAME in result.stdout diff --git a/tests/upgrades/test_errata.py b/tests/upgrades/test_errata.py index 0353c838e76..e35e8da634c 100644 --- a/tests/upgrades/test_errata.py +++ b/tests/upgrades/test_errata.py @@ -17,7 +17,6 @@ :Upstream: No """ import pytest -from broker import Broker from wait_for import wait_for from robottelo import constants @@ -222,9 +221,7 @@ def test_post_scenario_errata_count_installation(self, target_sat, pre_upgrade_d custom_repo_id = pre_upgrade_data.get('custom_repo_id') activation_key = pre_upgrade_data.get('activation_key') organization_id = pre_upgrade_data.get('organization_id') - rhel_client = Broker(host_class=ContentHost).from_inventory( - filter=f'@inv.hostname == "{client_hostname}"' - )[0] + rhel_client = ContentHost.get_host_by_hostname(client_hostname) custom_yum_repo = target_sat.api.Repository(id=custom_repo_id).read() host = target_sat.api.Host().search(query={'search': f'activation_key={activation_key}'})[0] assert host.id == rhel_client.nailgun_host.id, 'Host not found in Satellite' diff --git a/tests/upgrades/test_repository.py b/tests/upgrades/test_repository.py index 4a54bfbcab5..a2f5188669c 100644 --- a/tests/upgrades/test_repository.py +++ b/tests/upgrades/test_repository.py @@ -17,7 +17,6 @@ :Upstream: No """ import pytest -from broker import Broker from robottelo.config import settings from robottelo.constants import FAKE_0_CUSTOM_PACKAGE_NAME @@ -198,9 +197,7 @@ def test_post_scenario_custom_repo_check(self, target_sat, pre_upgrade_data): data={'environment_ids': lce_id} ) - rhel_client = Broker(host_class=ContentHost).from_inventory( - filter=f'@inv.hostname == "{client_hostname}"' - )[0] + rhel_client = ContentHost.get_host_by_hostname(client_hostname) result = rhel_client.execute(f'yum install -y {FAKE_4_CUSTOM_PACKAGE_NAME}') assert result.status == 0 @@ -299,9 +296,7 @@ def test_post_scenario_custom_repo_sca_toggle(self, pre_upgrade_data): org_name = pre_upgrade_data.get('org_name') product_name = pre_upgrade_data.get('product_name') repo_name = pre_upgrade_data.get('repo_name') - rhel_client = Broker(host_class=ContentHost).from_inventory( - filter=f'@inv.hostname == "{client_hostname}"' - )[0] + rhel_client = ContentHost.get_host_by_hostname(client_hostname) result = rhel_client.execute('subscription-manager repo-override --list') assert 'enabled: 1' in result.stdout assert f'{org_name}_{product_name}_{repo_name}' in result.stdout diff --git a/tests/upgrades/test_subscription.py b/tests/upgrades/test_subscription.py index b09f81e0521..f94e27692f6 100644 --- a/tests/upgrades/test_subscription.py +++ b/tests/upgrades/test_subscription.py @@ -17,11 +17,11 @@ :Upstream: No """ import pytest -from broker import Broker from manifester import Manifester from robottelo import constants from robottelo.config import settings +from robottelo.hosts import ContentHost class TestManifestScenarioRefresh: @@ -163,9 +163,7 @@ def test_post_subscription_scenario_auto_attach(self, request, target_sat, pre_u 1. Pre-upgrade content host should get subscribed. 2. All the cleanup should be completed successfully. """ - rhel_contenthost = Broker().from_inventory( - filter=f'@inv.hostname == "{pre_upgrade_data.rhel_client}"' - )[0] + rhel_contenthost = ContentHost.get_host_by_hostname(pre_upgrade_data.rhel_client) host = target_sat.api.Host().search(query={'search': f'name={rhel_contenthost.hostname}'})[ 0 ]