diff --git a/.github/workflows/auto_cherry_pick_merge.yaml b/.github/workflows/auto_cherry_pick_merge.yaml index 01896dd9d58..e0cb8e48173 100644 --- a/.github/workflows/auto_cherry_pick_merge.yaml +++ b/.github/workflows/auto_cherry_pick_merge.yaml @@ -67,7 +67,7 @@ jobs: - id: automerge name: Auto merge of cherry-picked PRs. - uses: "pascalgn/automerge-action@v0.16.0" + uses: "pascalgn/automerge-action@v0.16.2" env: GITHUB_TOKEN: "${{ secrets.CHERRYPICK_PAT }}" MERGE_LABELS: "AutoMerge_Cherry_Picked, Auto_Cherry_Picked" diff --git a/.github/workflows/dependency_merge.yml b/.github/workflows/dependency_merge.yml index ebac6d10c83..e2735e5a772 100644 --- a/.github/workflows/dependency_merge.yml +++ b/.github/workflows/dependency_merge.yml @@ -61,7 +61,7 @@ jobs: - id: automerge name: Auto merge of dependabot PRs. - uses: "pascalgn/automerge-action@v0.16.0" + uses: "pascalgn/automerge-action@v0.16.2" env: GITHUB_TOKEN: "${{ secrets.CHERRYPICK_PAT }}" MERGE_LABELS: "dependencies" diff --git a/conf/ui.yaml.template b/conf/ui.yaml.template index 750f715f7ed..2817b81120a 100644 --- a/conf/ui.yaml.template +++ b/conf/ui.yaml.template @@ -24,13 +24,15 @@ UI: WEBDRIVER: chrome # Binary location for selected wedriver (not needed if using saucelabs) WEBDRIVER_BINARY: /usr/bin/chromedriver - + RECORD_VIDEO: false + GRID_URL: http://infra-grid.example.com:4444 # Web_Kaifuku Settings (checkout https://github.com/RonnyPfannschmidt/webdriver_kaifuku) WEBKAIFUKU: webdriver: chrome/remote webdriver_options: command_executor: http://localhost:4444/wd/hub desired_capabilities: + se:recordVideo: '@jinja {{ this.ui.record_video }}' browserName: chrome chromeOptions: args: diff --git a/pytest_plugins/video_cleanup.py b/pytest_plugins/video_cleanup.py index 35c6fb5fb13..0eaeb570282 100644 --- a/pytest_plugins/video_cleanup.py +++ b/pytest_plugins/video_cleanup.py @@ -17,15 +17,16 @@ def _clean_video(session_id, test): - logger.info(f"cleaning up video files for session: {session_id} and test: {test}") + if settings.ui.record_video: + logger.info(f"cleaning up video files for session: {session_id} and test: {test}") - if settings.ui.grid_url and session_id: - grid = urlparse(url=settings.ui.grid_url) - infra_grid = Host(hostname=grid.hostname) - infra_grid.execute(command=f'rm -rf /var/www/html/videos/{session_id}') - logger.info(f"video cleanup for session {session_id} is complete") - else: - logger.warning("missing grid_url or session_id. unable to clean video files.") + if settings.ui.grid_url and session_id: + grid = urlparse(url=settings.ui.grid_url) + infra_grid = Host(hostname=grid.hostname) + infra_grid.execute(command=f'rm -rf /var/www/html/videos/{session_id}') + logger.info(f"video cleanup for session {session_id} is complete") + else: + logger.warning("missing grid_url or session_id. unable to clean video files.") def pytest_addoption(parser): diff --git a/requirements.txt b/requirements.txt index 34fe30f2a08..278091a8424 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ cryptography==41.0.7 deepdiff==6.7.1 dynaconf[vault]==3.2.4 fauxfactory==3.1.0 -jinja2==3.1.2 +jinja2==3.1.3 manifester==0.0.14 navmazing==1.2.2 productmd==1.38 diff --git a/robottelo/cli/hammer.py b/robottelo/cli/hammer.py index 661042050e6..cf14cd2f0d7 100644 --- a/robottelo/cli/hammer.py +++ b/robottelo/cli/hammer.py @@ -34,10 +34,23 @@ def _normalize_obj(obj): return obj +def is_csv(output): + """Verifies if the output string is eligible for converting into CSV""" + sniffer = csv.Sniffer() + try: + sniffer.sniff(output) + return True + except csv.Error: + return False + + def parse_csv(output): """Parse CSV output from Hammer CLI and convert it to python dictionary.""" # ignore warning about puppet and ostree deprecation output.replace('Puppet and OSTree will no longer be supported in Katello 3.16\n', '') + # Validate if the output is eligible for CSV conversions else return as it is + if not is_csv(output): + return output reader = csv.reader(output.splitlines()) # Generate the key names, spaces will be converted to dashes "-" keys = [_normalize(header) for header in next(reader)] diff --git a/robottelo/cli/repository.py b/robottelo/cli/repository.py index 85296f25e51..94fa8baa180 100644 --- a/robottelo/cli/repository.py +++ b/robottelo/cli/repository.py @@ -60,7 +60,7 @@ def synchronize(cls, options, return_raw_response=None, timeout=3600000): cls.command_sub = 'synchronize' return cls.execute( cls._construct_command(options), - output_format='csv', + output_format='base', ignore_stderr=True, return_raw_response=return_raw_response, timeout=timeout, diff --git a/robottelo/host_helpers/cli_factory.py b/robottelo/host_helpers/cli_factory.py index efb5a8fb4b8..fdfd29152ab 100644 --- a/robottelo/host_helpers/cli_factory.py +++ b/robottelo/host_helpers/cli_factory.py @@ -249,7 +249,6 @@ def __init__(self, satellite): self._satellite = satellite self.__dict__.update(initiate_repo_helpers(self._satellite)) - @lru_cache def __getattr__(self, name): """We intercept the usual attribute behavior on this class to emulate make_entity methods The keys in the dictionary above correspond to potential make_ methods diff --git a/robottelo/hosts.py b/robottelo/hosts.py index 8e48f963fe1..580c64fb3ef 100644 --- a/robottelo/hosts.py +++ b/robottelo/hosts.py @@ -1856,10 +1856,10 @@ def get_caller(): except Exception: raise finally: - video_url = settings.ui.grid_url.replace( - ':4444', f'/videos/{ui_session.ui_session_id}.mp4' - ) if self.record_property is not None and settings.ui.record_video: + video_url = settings.ui.grid_url.replace( + ':4444', f'/videos/{ui_session.ui_session_id}/video.mp4' + ) self.record_property('video_url', video_url) self.record_property('session_id', ui_session.ui_session_id) diff --git a/robottelo/utils/virtwho.py b/robottelo/utils/virtwho.py index 2f176d55341..5e0f3a4657a 100644 --- a/robottelo/utils/virtwho.py +++ b/robottelo/utils/virtwho.py @@ -506,7 +506,7 @@ def virtwho_package_locked(): assert "Packages are locked" in result[1] -def create_http_proxy(org, name=None, url=None, http_type='https'): +def create_http_proxy(org, location, name=None, url=None, http_type='https'): """ Creat a new http-proxy with attributes. :param name: Name of the proxy @@ -524,6 +524,7 @@ def create_http_proxy(org, name=None, url=None, http_type='https'): name=http_proxy_name, url=http_proxy_url, organization=[org.id], + location=[location.id], ).create() return http_proxy.url, http_proxy.name, http_proxy.id diff --git a/tests/foreman/api/test_discoveredhost.py b/tests/foreman/api/test_discoveredhost.py index df8444c951b..1bf31452aeb 100644 --- a/tests/foreman/api/test_discoveredhost.py +++ b/tests/foreman/api/test_discoveredhost.py @@ -201,7 +201,7 @@ def test_positive_provision_pxe_host( mac = provisioning_host._broker_args['provisioning_nic_mac_addr'] wait_for( lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [], - timeout=600, + timeout=1500, delay=40, ) discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0] @@ -251,7 +251,7 @@ def test_positive_provision_pxe_less_host( mac = pxeless_discovery_host._broker_args['provisioning_nic_mac_addr'] wait_for( lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [], - timeout=600, + timeout=1500, delay=40, ) discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0] @@ -386,7 +386,7 @@ def test_positive_reboot_pxe_host( mac = provisioning_host._broker_args['provisioning_nic_mac_addr'] wait_for( lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [], - timeout=240, + timeout=1500, delay=20, ) @@ -432,7 +432,7 @@ def test_positive_reboot_all_pxe_hosts( mac = host._broker_args['provisioning_nic_mac_addr'] wait_for( lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [], # noqa: B023 - timeout=240, + timeout=1500, delay=20, ) discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0] diff --git a/tests/foreman/api/test_provisioning.py b/tests/foreman/api/test_provisioning.py index 47dcc8d54a3..100a9964f9d 100644 --- a/tests/foreman/api/test_provisioning.py +++ b/tests/foreman/api/test_provisioning.py @@ -587,6 +587,7 @@ def test_rhel_pxe_provisioning_fips_enabled( @pytest.mark.e2e @pytest.mark.parametrize('pxe_loader', ['bios', 'uefi'], indirect=True) +@pytest.mark.on_premises_provisioning @pytest.mark.rhel_ver_match('[^6]') def test_capsule_pxe_provisioning( request, diff --git a/tests/foreman/cli/test_discoveredhost.py b/tests/foreman/cli/test_discoveredhost.py index 6da0330db17..5fbeca4ef42 100644 --- a/tests/foreman/cli/test_discoveredhost.py +++ b/tests/foreman/cli/test_discoveredhost.py @@ -52,7 +52,7 @@ def test_rhel_pxe_discovery_provisioning( wait_for( lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [], - timeout=600, + timeout=1500, delay=40, ) discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0] @@ -115,7 +115,7 @@ def test_rhel_pxeless_discovery_provisioning( wait_for( lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [], - timeout=600, + timeout=1500, delay=40, ) discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0] diff --git a/tests/foreman/cli/test_ping.py b/tests/foreman/cli/test_ping.py index 0a288143233..1036b3c5fd9 100644 --- a/tests/foreman/cli/test_ping.py +++ b/tests/foreman/cli/test_ping.py @@ -37,21 +37,13 @@ def test_positive_ping(target_sat, switch_user): """ result = target_sat.execute(f"su - {'postgres' if switch_user else 'root'} -c 'hammer ping'") assert result.stderr[1].decode() == '' - assert result.status == 0 - status_count = 0 - ok_count = 0 - # Exclude message from stdout for services candlepin_events and katello_events - result.stdout = [line for line in result.stdout.splitlines() if 'message' not in line] + # Filter lines containing status + statuses = [line for line in result.stdout.splitlines() if 'status:' in line.lower()] - # iterate over the lines grouping every 3 lines - # example [1, 2, 3, 4, 5, 6] will return [(1, 2, 3), (4, 5, 6)] - # only the status line is relevant for this test - for _, status, _ in zip(*[iter(result.stdout)] * 3, strict=False): # should this be strict? - status_count += 1 - - if status.split(':')[1].strip().lower() == 'ok': - ok_count += 1 + # Get count of total status lines and lines containing OK + status_count = len(statuses) + ok_count = len([status for status in statuses if status.split(':')[1].strip().lower() == 'ok']) if status_count == ok_count: assert result.status == 0, 'Return code should be 0 if all services are ok' diff --git a/tests/foreman/destructive/test_installer.py b/tests/foreman/destructive/test_installer.py index da213102b87..c130e0fcd20 100644 --- a/tests/foreman/destructive/test_installer.py +++ b/tests/foreman/destructive/test_installer.py @@ -11,6 +11,8 @@ :CaseImportance: Critical """ +import random + from fauxfactory import gen_domain, gen_string import pytest @@ -140,3 +142,40 @@ def test_positive_installer_certs_regenerate(target_sat): ) assert result.status == 0 assert 'FAIL' not in target_sat.cli.Base.ping() + + +def test_positive_installer_puma_worker_count(target_sat): + """Installer should set the puma worker count and thread max without having to manually + restart the foreman service. + + :id: d0e7d958-dd3e-4962-bf5a-8d7ec36f3485 + + :steps: + 1. Check how many puma workers there are + 2. Select a new worker count that is less than the default + 2. Change answer's file to have new count for puma workers + 3. Run satellite-installer --foreman-foreman-service-puma-workers new_count --foreman-foreman-service-puma-threads-max new_count + + :expectedresults: aux should show there are only new_count puma workers after installer runs + + :BZ: 2025760 + + :customerscenario: true + """ + count = int(target_sat.execute('pgrep --full "puma: cluster worker" | wc -l').stdout) + worker_count = str(random.randint(1, count - 1)) + result = target_sat.install( + InstallerCommand( + foreman_foreman_service_puma_workers=worker_count, + foreman_foreman_service_puma_threads_max=worker_count, + ) + ) + assert result.status == 0 + result = target_sat.execute(f'grep "foreman_service_puma_workers" {SATELLITE_ANSWER_FILE}') + assert worker_count in result.stdout + result = target_sat.execute('ps aux | grep -v grep | grep -e USER -e puma') + for i in range(count): + if i < int(worker_count): + assert f'cluster worker {i}' in result.stdout + else: + assert f'cluster worker {i}' not in result.stdout diff --git a/tests/foreman/ui/test_discoveredhost.py b/tests/foreman/ui/test_discoveredhost.py index ee437cfe897..292ad94f7a4 100644 --- a/tests/foreman/ui/test_discoveredhost.py +++ b/tests/foreman/ui/test_discoveredhost.py @@ -81,7 +81,7 @@ def test_positive_provision_pxe_host( mac = provisioning_host._broker_args['provisioning_nic_mac_addr'] wait_for( lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [], - timeout=240, + timeout=1500, delay=20, ) discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0] @@ -179,7 +179,7 @@ def test_positive_auto_provision_host_with_rule( mac = pxeless_discovery_host._broker_args['provisioning_nic_mac_addr'] wait_for( lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [], - timeout=240, + timeout=1500, delay=20, ) discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0] diff --git a/tests/foreman/virtwho/api/test_esx.py b/tests/foreman/virtwho/api/test_esx.py index cdea45e72b4..bb6046188a0 100644 --- a/tests/foreman/virtwho/api/test_esx.py +++ b/tests/foreman/virtwho/api/test_esx.py @@ -210,7 +210,7 @@ def test_positive_filter_option( @pytest.mark.tier2 def test_positive_proxy_option( - self, default_org, form_data_api, virtwho_config_api, target_sat + self, default_org, default_location, form_data_api, virtwho_config_api, target_sat ): """Verify http_proxy option by "PUT @@ -232,7 +232,7 @@ def test_positive_proxy_option( assert get_configure_option('no_proxy', ETC_VIRTWHO_CONFIG) == '*' # Check HTTTP Proxy and No_PROXY option http_proxy_url, http_proxy_name, http_proxy_id = create_http_proxy( - http_type='http', org=default_org + http_type='http', org=default_org, location=default_location ) no_proxy = 'test.satellite.com' virtwho_config_api.http_proxy_id = http_proxy_id @@ -245,7 +245,9 @@ def test_positive_proxy_option( assert get_configure_option('http_proxy', ETC_VIRTWHO_CONFIG) == http_proxy_url assert get_configure_option('no_proxy', ETC_VIRTWHO_CONFIG) == no_proxy # Check HTTTPs Proxy option - https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy(org=default_org) + https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy( + org=default_org, location=default_location + ) virtwho_config_api.http_proxy_id = https_proxy_id virtwho_config_api.update(['http_proxy_id']) deploy_configure_by_command( diff --git a/tests/foreman/virtwho/api/test_esx_sca.py b/tests/foreman/virtwho/api/test_esx_sca.py index 290e67242f5..2d43f59637a 100644 --- a/tests/foreman/virtwho/api/test_esx_sca.py +++ b/tests/foreman/virtwho/api/test_esx_sca.py @@ -227,7 +227,9 @@ def test_positive_filter_option( assert result.exclude_host_parents == regex @pytest.mark.tier2 - def test_positive_proxy_option(self, module_sca_manifest_org, form_data_api, target_sat): + def test_positive_proxy_option( + self, module_sca_manifest_org, default_location, form_data_api, target_sat + ): """Verify http_proxy option by "PUT /foreman_virt_who_configure/api/v2/configs/:id"" @@ -251,7 +253,7 @@ def test_positive_proxy_option(self, module_sca_manifest_org, form_data_api, tar assert get_configure_option('no_proxy', ETC_VIRTWHO_CONFIG) == '*' # Check HTTTP Proxy and No_PROXY option http_proxy_url, http_proxy_name, http_proxy_id = create_http_proxy( - http_type='http', org=module_sca_manifest_org + http_type='http', org=module_sca_manifest_org, location=default_location ) no_proxy = 'test.satellite.com' virtwho_config.http_proxy_id = http_proxy_id @@ -269,7 +271,7 @@ def test_positive_proxy_option(self, module_sca_manifest_org, form_data_api, tar assert result.no_proxy == no_proxy # Check HTTTPs Proxy option https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy( - org=module_sca_manifest_org + org=module_sca_manifest_org, location=default_location ) virtwho_config.http_proxy_id = https_proxy_id virtwho_config.update(['http_proxy_id']) diff --git a/tests/foreman/virtwho/cli/test_esx.py b/tests/foreman/virtwho/cli/test_esx.py index c661d31cd67..c18ffb3ab30 100644 --- a/tests/foreman/virtwho/cli/test_esx.py +++ b/tests/foreman/virtwho/cli/test_esx.py @@ -214,7 +214,7 @@ def test_positive_filter_option( @pytest.mark.tier2 def test_positive_proxy_option( - self, default_org, form_data_cli, virtwho_config_cli, target_sat + self, default_org, default_location, form_data_cli, virtwho_config_cli, target_sat ): """Verify http_proxy option by hammer virt-who-config update" @@ -227,7 +227,9 @@ def test_positive_proxy_option( :BZ: 1902199 """ # Check the https proxy option, update it via http proxy name - https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy(org=default_org) + https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy( + org=default_org, location=default_location + ) no_proxy = 'test.satellite.com' target_sat.cli.VirtWhoConfig.update( {'id': virtwho_config_cli['id'], 'http-proxy': https_proxy_name, 'no-proxy': no_proxy} @@ -244,7 +246,7 @@ def test_positive_proxy_option( # Check the http proxy option, update it via http proxy id http_proxy_url, http_proxy_name, http_proxy_id = create_http_proxy( - http_type='http', org=default_org + http_type='http', org=default_org, location=default_location ) target_sat.cli.VirtWhoConfig.update( {'id': virtwho_config_cli['id'], 'http-proxy-id': http_proxy_id} diff --git a/tests/foreman/virtwho/cli/test_esx_sca.py b/tests/foreman/virtwho/cli/test_esx_sca.py index 7c5a4f5bc20..6b15b6803f9 100644 --- a/tests/foreman/virtwho/cli/test_esx_sca.py +++ b/tests/foreman/virtwho/cli/test_esx_sca.py @@ -250,7 +250,12 @@ def test_positive_filter_option( @pytest.mark.tier2 def test_positive_proxy_option( - self, module_sca_manifest_org, form_data_cli, virtwho_config_cli, target_sat + self, + module_sca_manifest_org, + default_location, + form_data_cli, + virtwho_config_cli, + target_sat, ): """Verify http_proxy option by hammer virt-who-config update" @@ -266,7 +271,7 @@ def test_positive_proxy_option( """ # Check the https proxy option, update it via http proxy name https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy( - org=module_sca_manifest_org + org=module_sca_manifest_org, location=default_location ) no_proxy = 'test.satellite.com' target_sat.cli.VirtWhoConfig.update( @@ -284,7 +289,7 @@ def test_positive_proxy_option( # Check the http proxy option, update it via http proxy id http_proxy_url, http_proxy_name, http_proxy_id = create_http_proxy( - http_type='http', org=module_sca_manifest_org + http_type='http', org=module_sca_manifest_org, location=default_location ) target_sat.cli.VirtWhoConfig.update( {'id': virtwho_config_cli['id'], 'http-proxy-id': http_proxy_id} diff --git a/tests/foreman/virtwho/ui/test_esx.py b/tests/foreman/virtwho/ui/test_esx.py index b81536ab211..efd46e6deba 100644 --- a/tests/foreman/virtwho/ui/test_esx.py +++ b/tests/foreman/virtwho/ui/test_esx.py @@ -215,7 +215,9 @@ def test_positive_filtering_option( assert regex == get_configure_option('exclude_host_parents', config_file) @pytest.mark.tier2 - def test_positive_proxy_option(self, default_org, virtwho_config_ui, org_session, form_data_ui): + def test_positive_proxy_option( + self, default_org, default_location, virtwho_config_ui, org_session, form_data_ui + ): """Verify 'HTTP Proxy' and 'Ignore Proxy' options. :id: 6659d577-0135-4bf0-81af-14b930011536 @@ -225,9 +227,11 @@ def test_positive_proxy_option(self, default_org, virtwho_config_ui, org_session :CaseImportance: Medium """ - https_proxy, https_proxy_name, https_proxy_id = create_http_proxy(org=default_org) + https_proxy, https_proxy_name, https_proxy_id = create_http_proxy( + org=default_org, location=default_location + ) http_proxy, http_proxy_name, http_proxy_id = create_http_proxy( - http_type='http', org=default_org + http_type='http', org=default_org, location=default_location ) name = form_data_ui['name'] config_id = get_configure_id(name) @@ -547,7 +551,9 @@ def test_positive_virtwho_manager_role( assert not org_session.user.search(username) @pytest.mark.tier2 - def test_positive_overview_label_name(self, default_org, form_data_ui, org_session): + def test_positive_overview_label_name( + self, default_org, default_location, form_data_ui, org_session + ): """Verify the label name on virt-who config Overview Page. :id: 21df8175-bb41-422e-a263-8677bc3a9565 @@ -561,7 +567,9 @@ def test_positive_overview_label_name(self, default_org, form_data_ui, org_sessi name = gen_string('alpha') form_data_ui['name'] = name hypervisor_type = form_data_ui['hypervisor_type'] - http_proxy_url, proxy_name, proxy_id = create_http_proxy(org=default_org) + http_proxy_url, proxy_name, proxy_id = create_http_proxy( + org=default_org, location=default_location + ) form_data_ui['proxy'] = http_proxy_url form_data_ui['no_proxy'] = 'test.satellite.com' regex = '.*redhat.com'