From e5678da98ab6fcf96e63fc3ab994e919209473a1 Mon Sep 17 00:00:00 2001 From: Jitendra Yejare Date: Fri, 14 Jun 2024 02:35:32 +0530 Subject: [PATCH] Enabling Robottelo for IPv6 Testing (#14160) * Enabling Robottelo for IPv6 Testing * Enabling broker context managers for ipv6 network * Translating URLs for Ipv6 * Sanity Testing fixes for Ipv6 * Review Fixes * Removed the explicit http proxy teardowns * Git checks fixes --- conf/capsule.yaml.template | 1 + conf/dynaconf_hooks.py | 3 +- conf/server.yaml.template | 5 + pytest_fixtures/component/maintain.py | 4 +- pytest_fixtures/core/broker.py | 3 + pytest_fixtures/core/sat_cap_factory.py | 18 ++- pytest_fixtures/core/xdist.py | 5 +- robottelo/config/validators.py | 2 + robottelo/host_helpers/contenthost_mixins.py | 11 +- robottelo/hosts.py | 113 ++++++++++++++++++- robottelo/utils/ohsnap.py | 17 +-- robottelo/utils/url.py | 41 +++++++ tests/foreman/installer/test_installer.py | 12 +- 13 files changed, 210 insertions(+), 25 deletions(-) diff --git a/conf/capsule.yaml.template b/conf/capsule.yaml.template index b618bf57ea9..4f50d26bfaf 100644 --- a/conf/capsule.yaml.template +++ b/conf/capsule.yaml.template @@ -17,3 +17,4 @@ CAPSULE: OS: deploy-rhel # workflow to deploy OS that is ready to run the product # Dictionary of arguments which should be passed along to the deploy workflow DEPLOY_ARGUMENTS: + # deploy_network_type: '@jinja {{"ipv6" if this.server.is_ipv6 else "ipv4"}}' diff --git a/conf/dynaconf_hooks.py b/conf/dynaconf_hooks.py index 85baf7d52cb..b3d7a8e219c 100644 --- a/conf/dynaconf_hooks.py +++ b/conf/dynaconf_hooks.py @@ -7,7 +7,7 @@ from robottelo.logging import logger from robottelo.utils.ohsnap import dogfood_repository -from robottelo.utils.url import is_url +from robottelo.utils.url import ipv6_hostname_translation, is_url def post(settings): @@ -26,6 +26,7 @@ def post(settings): ) data = get_repos_config(settings) write_cache(settings_cache_path, data) + ipv6_hostname_translation(settings, data) config_migrations(settings, data) data['dynaconf_merge'] = True return data diff --git a/conf/server.yaml.template b/conf/server.yaml.template index c5b844e6509..e196dd9413d 100644 --- a/conf/server.yaml.template +++ b/conf/server.yaml.template @@ -13,6 +13,10 @@ SERVER: SOURCE: "internal" # The RHEL Base OS Version(x.y) where the Satellite is installed RHEL_VERSION: '7' + # If the the satellite server is IPv6 server + IS_IPV6: False + # HTTP Proxy url for IPv6 satellite to connect for outer world access + HTTP_PROXY_IPv6_URL: # run-on-one - All xdist runners default to the first satellite # balance - xdist runners will be split between available satellites # on-demand - any xdist runner without a satellite will have a new one provisioned. @@ -32,6 +36,7 @@ SERVER: OS: deploy-rhel # workflow to deploy OS that is ready to run the product # Dictionary of arguments which should be passed along to the deploy workflow # DEPLOY_ARGUMENTS: + # deploy_network_type: '@jinja {{"ipv6" if this.server.is_ipv6 else "ipv4"}}' # HTTP scheme when building the server URL # Suggested values for "scheme" are "http" and "https". SCHEME: https diff --git a/pytest_fixtures/component/maintain.py b/pytest_fixtures/component/maintain.py index 68aa82f5c86..1e0e264e0ca 100644 --- a/pytest_fixtures/component/maintain.py +++ b/pytest_fixtures/component/maintain.py @@ -24,7 +24,9 @@ def module_stash(request): @pytest.fixture(scope='module') def sat_maintain(request, module_target_sat, module_capsule_configured): if settings.remotedb.server: - yield Satellite(settings.remotedb.server) + sat = Satellite(settings.remotedb.server) + sat.enable_ipv6_http_proxy() + yield sat else: module_target_sat.register_to_cdn(pool_ids=settings.subscription.fm_rhn_poolid.split()) hosts = {'satellite': module_target_sat, 'capsule': module_capsule_configured} diff --git a/pytest_fixtures/core/broker.py b/pytest_fixtures/core/broker.py index f3356dbbef5..9e63ce92a14 100644 --- a/pytest_fixtures/core/broker.py +++ b/pytest_fixtures/core/broker.py @@ -24,6 +24,7 @@ def _target_sat_imp(request, _default_sat, satellite_factory): """This is the actual working part of the following target_sat fixtures""" if request.node.get_closest_marker(name='destructive'): new_sat = satellite_factory() + new_sat.enable_ipv6_http_proxy() yield new_sat new_sat.teardown() Broker(hosts=[new_sat]).checkin() @@ -32,6 +33,8 @@ def _target_sat_imp(request, _default_sat, satellite_factory): settings.set('server.hostname', installer_sat.hostname) yield installer_sat else: + if _default_sat: + _default_sat.enable_ipv6_http_proxy() yield _default_sat diff --git a/pytest_fixtures/core/sat_cap_factory.py b/pytest_fixtures/core/sat_cap_factory.py index 3cfebdee160..7eba78ac14b 100644 --- a/pytest_fixtures/core/sat_cap_factory.py +++ b/pytest_fixtures/core/sat_cap_factory.py @@ -31,6 +31,7 @@ def resolve_deploy_args(args_dict): def _target_satellite_host(request, satellite_factory): if 'sanity' not in request.config.option.markexpr: new_sat = satellite_factory() + new_sat.enable_ipv6_http_proxy() yield new_sat new_sat.teardown() Broker(hosts=[new_sat]).checkin() @@ -48,6 +49,7 @@ def cached_capsule_cdn_register(hostname=None): def _target_capsule_host(request, capsule_factory): if 'sanity' not in request.config.option.markexpr and not request.config.option.n_minus: new_cap = capsule_factory() + new_cap.enable_ipv6_http_proxy() yield new_cap new_cap.teardown() Broker(hosts=[new_cap]).checkin() @@ -94,6 +96,7 @@ def factory(retry_limit=3, delay=300, workflow=None, **broker_args): def large_capsule_host(capsule_factory): """A fixture that provides a Capsule based on config settings""" new_cap = capsule_factory(deploy_flavor=settings.flavors.custom_db) + new_cap.enable_ipv6_http_proxy() yield new_cap new_cap.teardown() Broker(hosts=[new_cap]).checkin() @@ -244,6 +247,7 @@ def module_lb_capsule(retry_limit=3, delay=300, **broker_args): ) cap_hosts = wait_for(hosts.checkout, timeout=timeout, delay=delay) + [cap.enable_ipv6_http_proxy() for cap in cap_hosts.out] yield cap_hosts.out [cap.teardown() for cap in cap_hosts.out] @@ -278,6 +282,7 @@ def parametrized_enrolled_sat( ): """Yields a Satellite enrolled into [IDM, AD] as parameter.""" new_sat = satellite_factory() + new_sat.enable_ipv6_http_proxy() ipa_host = IPAHost(new_sat) new_sat.register_to_cdn() if 'IDM' in request.param: @@ -297,6 +302,7 @@ def get_deploy_args(request): rhel_version = get_sat_rhel_version() deploy_args = { 'deploy_rhel_version': rhel_version.base_version, + 'deploy_network_type': 'ipv6' if settings.server.is_ipv6 else 'ipv4', 'deploy_flavor': settings.flavors.default, 'promtail_config_template_file': 'config_sat.j2', 'workflow': settings.server.deploy_workflows.os, @@ -328,11 +334,13 @@ def cap_ready_rhel(): rhel_version = Version(settings.capsule.version.rhel_version) deploy_args = { 'deploy_rhel_version': rhel_version.base_version, + 'deploy_network_type': 'ipv6' if settings.server.is_ipv6 else 'ipv4', 'deploy_flavor': settings.flavors.default, 'promtail_config_template_file': 'config_sat.j2', 'workflow': settings.capsule.deploy_workflows.os, } with Broker(**deploy_args, host_class=Capsule) as host: + host.enable_ipv6_http_proxy() yield host @@ -352,7 +360,7 @@ def installer_satellite(request): sat = lru_sat_ready_rhel(getattr(request, 'param', None)) sat.setup_firewall() # # Register for RHEL8 repos, get Ohsnap repofile, and enable and download satellite - sat.register_to_cdn() + sat.register_to_cdn(enable_proxy=True) sat.download_repofile( product='satellite', release=settings.server.version.release, @@ -371,12 +379,12 @@ def installer_satellite(request): ).get_command(), timeout='30m', ) + sat.enable_ipv6_http_proxy() if 'sanity' in request.config.option.markexpr: configure_nailgun() configure_airgun() yield sat if 'sanity' not in request.config.option.markexpr: - sanity_sat = Satellite(sat.hostname) - sanity_sat.unregister() - broker_sat = Satellite.get_host_by_hostname(sanity_sat.hostname) - Broker(hosts=[broker_sat]).checkin() + sat = Satellite.get_host_by_hostname(sat.hostname) + sat.unregister() + Broker(hosts=[sat]).checkin() diff --git a/pytest_fixtures/core/xdist.py b/pytest_fixtures/core/xdist.py index be497dd988d..216589e85d7 100644 --- a/pytest_fixtures/core/xdist.py +++ b/pytest_fixtures/core/xdist.py @@ -19,8 +19,9 @@ def align_to_satellite(request, worker_id, satellite_factory): if settings.server.hostname: sanity_sat = Satellite(settings.server.hostname) sanity_sat.unregister() - broker_sat = Satellite.get_host_by_hostname(sanity_sat.hostname) - Broker(hosts=[broker_sat]).checkin() + if settings.server.auto_checkin: + broker_sat = Satellite.get_host_by_hostname(sanity_sat.hostname) + Broker(hosts=[broker_sat]).checkin() else: # clear any hostname that may have been previously set settings.set("server.hostname", None) diff --git a/robottelo/config/validators.py b/robottelo/config/validators.py index 9380cbb5267..1c37d63404f 100644 --- a/robottelo/config/validators.py +++ b/robottelo/config/validators.py @@ -35,6 +35,8 @@ Validator('server.ssh_username', default='root'), Validator('server.ssh_password', default=None), Validator('server.verify_ca', default=False), + Validator('server.is_ipv6', is_type_of=bool, default=False), + Validator('server.http_proxy_ipv6_url', is_type_of=str, default=None), ], content_host=[ Validator('content_host.default_rhel_version', must_exist=True), diff --git a/robottelo/host_helpers/contenthost_mixins.py b/robottelo/host_helpers/contenthost_mixins.py index c76d23405fe..2fcfd909d3e 100644 --- a/robottelo/host_helpers/contenthost_mixins.py +++ b/robottelo/host_helpers/contenthost_mixins.py @@ -90,11 +90,16 @@ def _dogfood_helper(self, product, release, repo=None): ) return product, release, v_major, repo - def download_repofile(self, product=None, release=None, snap=''): + def download_repofile(self, product=None, release=None, snap='', proxy=None): """Downloads the tools/client, capsule, or satellite repos on the machine""" product, release, v_major, _ = self._dogfood_helper(product, release) - url = dogfood_repofile_url(settings.ohsnap, product, release, v_major, snap) - self.execute(f'curl -o /etc/yum.repos.d/dogfood.repo -L {url}') + if not proxy and settings.server.is_ipv6: + proxy = settings.server.http_proxy_ipv6_url + url = dogfood_repofile_url(settings.ohsnap, product, release, v_major, snap, proxy=proxy) + command = f'curl -o /etc/yum.repos.d/dogfood.repo -L {url}' + if settings.server.is_ipv6: + command += f' -x {settings.server.http_proxy_ipv6_url}' + self.execute(command) def dogfood_repository(self, repo=None, product=None, release=None, snap=''): """Returns a repository definition based on the arguments provided""" diff --git a/robottelo/hosts.py b/robottelo/hosts.py index 0c38e61735b..68c789d050c 100644 --- a/robottelo/hosts.py +++ b/robottelo/hosts.py @@ -73,6 +73,7 @@ def lru_sat_ready_rhel(rhel_ver): rhel_version = rhel_ver or settings.server.version.rhel_version deploy_args = { 'deploy_rhel_version': rhel_version, + 'deploy_network_type': 'ipv6' if settings.server.is_ipv6 else 'ipv4', 'deploy_flavor': settings.flavors.default, 'promtail_config_template_file': 'config_sat.j2', 'workflow': settings.server.deploy_workflows.os, @@ -823,6 +824,7 @@ def register_contenthost( auto_attach=False, serverurl=None, baseurl=None, + enable_proxy=False, ): """Registers content host on foreman server either by specifying organization name and activation key name or by specifying organization @@ -879,6 +881,7 @@ def register_contenthost( cmd += f' --serverurl {serverurl}' if baseurl: cmd += f' --baseurl {baseurl}' + return self.execute(cmd) def unregister(self): @@ -925,6 +928,13 @@ def put_ssh_key(self, source_key_path, destination_key_name): if result.status != 0: raise CLIFactoryError(f'Failed to chmod ssh key file:\n{result.stderr}') + def enable_rhsm_proxy(self, hostname, port=None): + """Configures proxy for subscription manager""" + cmd = f"subscription-manager config --server.proxy_hostname={hostname}" + if port: + cmd += f' --server.proxy_port={port}' + self.execute(cmd) + def add_authorized_key(self, pub_key): """Inject a public key into the authorized keys file @@ -1469,7 +1479,7 @@ def install_tracer(self): raise ContentHostError('There was an error installing katello-host-tools-tracer') self.execute('katello-tracer-upload') - def register_to_cdn(self, pool_ids=None): + def register_to_cdn(self, pool_ids=None, enable_proxy=False): """Subscribe satellite to CDN""" if pool_ids is None: pool_ids = [settings.subscription.rhn_poolid] @@ -1479,6 +1489,7 @@ def register_to_cdn(self, pool_ids=None): lce=None, username=settings.subscription.rhn_username, password=settings.subscription.rhn_password, + enable_proxy=enable_proxy, ) if cmd_result.status != 0: raise ContentHostError( @@ -1633,6 +1644,17 @@ def enable_capsule_downstream_repos(self): snap=settings.capsule.version.snap, ) + def enable_ipv6_http_proxy(self): + """Execute procedures for enabling IPv6 HTTP Proxy on Capsule using SM""" + if all([settings.server.is_ipv6, settings.server.http_proxy_ipv6_url]): + url = urlparse(settings.server.http_proxy_ipv6_url) + self.enable_rhsm_proxy(url.hostname, url.port) + + def disable_ipv6_http_proxy(self): + """Executes procedures for disabling IPv6 HTTP Proxy on Capsule""" + if settings.server.is_ipv6: + self.execute('subscription-manager remove server.proxy_hostname server.proxy_port') + def capsule_setup(self, sat_host=None, capsule_cert_opts=None, **installer_kwargs): """Prepare the host and run the capsule installer""" self._satellite = sat_host or Satellite() @@ -1751,6 +1773,29 @@ def cli(self): self._cli._configured = True return self._cli + def enable_satellite_or_capsule_module_for_rhel8(self): + """Enable Satellite/Capsule module for RHEL8. + Note: Make sure required repos are enabled before using this. + """ + if self.os_version.major == 8: + if settings.server.is_ipv6: + self.execute( + f"echo -e 'proxy={settings.server.http_proxy_ipv6_url}' >> /etc/dnf/dnf.conf" + ) + assert ( + self.execute( + f'dnf -y module enable {self.product_rpm_name}:el{self.os_version.major}' + ).status + == 0 + ) + + def install_satellite_or_capsule_package(self): + """Install Satellite/Capsule package. Also handles module enablement for RHEL8. + Note: Make sure required repos are enabled before using this. + """ + self.enable_satellite_or_capsule_module_for_rhel8() + assert self.execute(f'dnf -y install {self.product_rpm_name}').status == 0 + class Satellite(Capsule, SatelliteMixins): product_rpm_name = 'satellite' @@ -1779,6 +1824,55 @@ def _swap_nailgun(self, new_version): to_clear = [k for k in sys.modules if 'nailgun' in k] [sys.modules.pop(k) for k in to_clear] + def enable_ipv6_http_proxy(self): + """Execute procedures for enabling IPv6 HTTP Proxy""" + if not all([settings.server.is_ipv6, settings.server.http_proxy_ipv6_url]): + logger.warning( + 'The IPv6 HTTP Proxy setting is not enabled. Skipping the IPv6 HTTP Proxy setup.' + ) + return None + proxy_name = 'Robottelo IPv6 Automation Proxy' + if not self.cli.HttpProxy.exists(search=('name', proxy_name)): + http_proxy = self.api.HTTPProxy( + name=proxy_name, url=settings.server.http_proxy_ipv6_url + ).create() + else: + logger.info( + 'The IPv6 HTTP Proxy is already enabled. Skipping the IPv6 HTTP Proxy setup.' + ) + http_proxy = self.api.HTTPProxy().search(query={'search': f'name={proxy_name}'})[0] + # Setting HTTP Proxy as default in the settings + self.cli.Settings.set( + { + 'name': 'content_default_http_proxy', + 'value': proxy_name, + } + ) + self.cli.Settings.set( + { + 'name': 'http_proxy', + 'value': settings.server.http_proxy_ipv6_url, + } + ) + return http_proxy + + def disable_ipv6_http_proxy(self, http_proxy): + """Execute procedures for disabling IPv6 HTTP Proxy""" + if http_proxy: + http_proxy.delete() + self.cli.Settings.set( + { + 'name': 'content_default_http_proxy', + 'value': '', + } + ) + self.cli.Settings.set( + { + 'name': 'http_proxy', + 'value': '', + } + ) + @property def api(self): """Import all nailgun entities and wrap them under self.api""" @@ -2307,6 +2401,23 @@ def sync_inventory_status(self, org): ) return inventory_sync + def register_contenthost( + self, + org='Default_Organization', + lce='Library', + username=settings.server.admin_username, + password=settings.server.admin_password, + enable_proxy=False, + ): + """Satellite Registration to CDN""" + # Enabling proxy for IPv6 + if enable_proxy and all([settings.server.is_ipv6, settings.server.http_proxy_ipv6_url]): + url = urlparse(settings.server.http_proxy_ipv6_url) + self.enable_rhsm_proxy(url.hostname, url.port) + return super().register_contenthost( + org=org, lce=lce, username=username, password=password, enable_proxy=enable_proxy + ) + class SSOHost(Host): """Class for RHSSO functions and setup""" diff --git a/robottelo/utils/ohsnap.py b/robottelo/utils/ohsnap.py index 83669cdb61e..7e870888656 100644 --- a/robottelo/utils/ohsnap.py +++ b/robottelo/utils/ohsnap.py @@ -21,7 +21,7 @@ def ohsnap_response_hook(r, *args, **kwargs): r.raise_for_status() -def ohsnap_repo_url(ohsnap, request_type, product, release, os_release, snap=''): +def ohsnap_repo_url(ohsnap, request_type, product, release, os_release, snap='', proxy=None): """Returns a URL pointing to Ohsnap "repo_file" or "repositories" API endpoint""" if request_type not in ['repo_file', 'repositories']: raise InvalidArgumentError('Type must be one of "repo_file" or "repositories"') @@ -42,11 +42,14 @@ def ohsnap_repo_url(ohsnap, request_type, product, release, os_release, snap='') f'.z version component not provided in the release ({release}),' f' fetching the recent z-stream from ohsnap' ) + request_query = { + 'url': f'{ohsnap.host}/api/streams', + 'hooks': {'response': ohsnap_response_hook}, + } + if proxy: + request_query['proxies'] = {'http': proxy} res, _ = wait_for( - lambda: requests.get( - f'{ohsnap.host}/api/streams', - hooks={'response': ohsnap_response_hook}, - ), + lambda: requests.get(**request_query), handle_exception=True, raise_original=True, timeout=ohsnap.request_retry.timeout, @@ -69,8 +72,8 @@ def ohsnap_repo_url(ohsnap, request_type, product, release, os_release, snap='') ) -def dogfood_repofile_url(ohsnap, product, release, os_release, snap=''): - return ohsnap_repo_url(ohsnap, 'repo_file', product, release, os_release, snap) +def dogfood_repofile_url(ohsnap, product, release, os_release, snap='', proxy=None): + return ohsnap_repo_url(ohsnap, 'repo_file', product, release, os_release, snap, proxy) def dogfood_repository( diff --git a/robottelo/utils/url.py b/robottelo/utils/url.py index cadcfb56d14..a84a85d4c1c 100644 --- a/robottelo/utils/url.py +++ b/robottelo/utils/url.py @@ -1,5 +1,7 @@ from urllib.parse import urlparse +from robottelo.logging import logger + def is_url(url): try: @@ -7,3 +9,42 @@ def is_url(url): return all([result.scheme, result.netloc]) except (ValueError, AttributeError): return False + + +def is_ipv4_url(text): + """Verify if the URL is IPv4 url""" + return isinstance(text, str) and 'ipv4' in text and 'redhat.com' in text + + +def ipv6_translator(settings_list, setting_major, data): + """Translates the hostname containing IPv4 to IPv6 and updates the settings object""" + dotted_settings = '.'.join(setting_major) + for _key, _val in settings_list.items(): + if is_ipv4_url(_val): + data[f'{dotted_settings}.{_key}'] = str(_val).replace('ipv4', 'ipv6') + logger.debug(f'Setting translated to IPv6, Path: {dotted_settings}.{_key}') + elif isinstance(_val, list): + updated = False + new_list = _val + for i in range(len(new_list)): + if is_ipv4_url(new_list[i]): + new_list[i] = new_list[i].replace('ipv4', 'ipv6') + updated = True + if updated: + data[f'{dotted_settings}.{_key}'] = new_list + logger.debug(f'Setting translated to IPv6, Path: {dotted_settings}.{_key}') + elif isinstance(_val, dict): + new_setting_major = setting_major + [_key] + ipv6_translator(settings_list=_val, setting_major=new_setting_major, data=data) + + +def ipv6_hostname_translation(settings, data): + """Migrates any IPv4 containing hostname in conf to IPv6 hostname""" + settings_path = [] + if settings.server.is_ipv6: + all_settings = settings.loaded_by_loaders.items() + for loader_name, loader_settings in tuple(all_settings): + if loader_name.loader == 'yaml': + ipv6_translator(loader_settings, settings_path, data) + else: + logger.debug('IPv6 Hostname dynaconf migration hook is skipped for IPv4 testing') diff --git a/tests/foreman/installer/test_installer.py b/tests/foreman/installer/test_installer.py index 1bc65fd5ab3..e83e512129f 100644 --- a/tests/foreman/installer/test_installer.py +++ b/tests/foreman/installer/test_installer.py @@ -1382,7 +1382,9 @@ def sat_default_install(module_sat_ready_rhels): f'foreman-initial-admin-password {settings.server.admin_password}', ] install_satellite(module_sat_ready_rhels[0], installer_args) - return module_sat_ready_rhels[0] + sat = module_sat_ready_rhels[0] + sat.enable_ipv6_http_proxy() + return sat @pytest.fixture(scope='module') @@ -1397,10 +1399,10 @@ def sat_non_default_install(module_sat_ready_rhels): 'foreman-proxy-plugin-discovery-install-images true', ] install_satellite(module_sat_ready_rhels[1], installer_args, enable_fapolicyd=True) - module_sat_ready_rhels[1].execute( - 'dnf -y --disableplugin=foreman-protector install foreman-discovery-image' - ) - return module_sat_ready_rhels[1] + sat = module_sat_ready_rhels[1] + sat.enable_ipv6_http_proxy() + sat.execute('dnf -y --disableplugin=foreman-protector install foreman-discovery-image') + return sat @pytest.mark.e2e