Merge branch 'master' into py312

.github/workflows/auto_cherry_pick_merge.yaml

@@ -67,7 +67,7 @@ jobs:
 
       - id: automerge
         name: Auto merge of cherry-picked PRs.
-        uses: "pascalgn/[email protected].0"
+        uses: "pascalgn/[email protected].2"
         env:
           GITHUB_TOKEN: "${{ secrets.CHERRYPICK_PAT }}"
           MERGE_LABELS: "AutoMerge_Cherry_Picked, Auto_Cherry_Picked"

.github/workflows/dependency_merge.yml

@@ -61,7 +61,7 @@ jobs:
 
       - id: automerge
         name: Auto merge of dependabot PRs.
-        uses: "pascalgn/[email protected].0"
+        uses: "pascalgn/[email protected].2"
         env:
           GITHUB_TOKEN: "${{ secrets.CHERRYPICK_PAT }}"
           MERGE_LABELS: "dependencies"

conf/ui.yaml.template

@@ -24,13 +24,15 @@ UI:
   WEBDRIVER: chrome
   # Binary location for selected wedriver (not needed if using saucelabs)
   WEBDRIVER_BINARY: /usr/bin/chromedriver
-
+  RECORD_VIDEO: false
+  GRID_URL: http://infra-grid.example.com:4444
   # Web_Kaifuku Settings (checkout https://github.com/RonnyPfannschmidt/webdriver_kaifuku)
   WEBKAIFUKU:
     webdriver: chrome/remote
     webdriver_options:
       command_executor: http://localhost:4444/wd/hub
       desired_capabilities:
+        se:recordVideo: '@jinja {{ this.ui.record_video }}'
         browserName: chrome
         chromeOptions:
           args:

pytest_plugins/video_cleanup.py

@@ -17,15 +17,16 @@
 
 
 def _clean_video(session_id, test):
-    logger.info(f"cleaning up video files for session: {session_id} and test: {test}")
+    if settings.ui.record_video:
+        logger.info(f"cleaning up video files for session: {session_id} and test: {test}")
 
-    if settings.ui.grid_url and session_id:
-        grid = urlparse(url=settings.ui.grid_url)
-        infra_grid = Host(hostname=grid.hostname)
-        infra_grid.execute(command=f'rm -rf /var/www/html/videos/{session_id}')
-        logger.info(f"video cleanup for session {session_id} is complete")
-    else:
-        logger.warning("missing grid_url or session_id. unable to clean video files.")
+        if settings.ui.grid_url and session_id:
+            grid = urlparse(url=settings.ui.grid_url)
+            infra_grid = Host(hostname=grid.hostname)
+            infra_grid.execute(command=f'rm -rf /var/www/html/videos/{session_id}')
+            logger.info(f"video cleanup for session {session_id} is complete")
+        else:
+            logger.warning("missing grid_url or session_id. unable to clean video files.")
 
 
 def pytest_addoption(parser):

requirements.txt

@@ -6,7 +6,7 @@ cryptography==41.0.7
 deepdiff==6.7.1
 dynaconf[vault]==3.2.4
 fauxfactory==3.1.0
-jinja2==3.1.2
+jinja2==3.1.3
 manifester==0.0.14
 navmazing==1.2.2
 productmd==1.38

robottelo/cli/hammer.py

@@ -34,10 +34,23 @@ def _normalize_obj(obj):
     return obj
 
 
+def is_csv(output):
+    """Verifies if the output string is eligible for converting into CSV"""
+    sniffer = csv.Sniffer()
+    try:
+        sniffer.sniff(output)
+        return True
+    except csv.Error:
+        return False
+
+
 def parse_csv(output):
     """Parse CSV output from Hammer CLI and convert it to python dictionary."""
     # ignore warning about puppet and ostree deprecation
     output.replace('Puppet and OSTree will no longer be supported in Katello 3.16\n', '')
+    # Validate if the output is eligible for CSV conversions else return as it is
+    if not is_csv(output):
+        return output
     reader = csv.reader(output.splitlines())
     # Generate the key names, spaces will be converted to dashes "-"
     keys = [_normalize(header) for header in next(reader)]

robottelo/cli/repository.py

@@ -60,7 +60,7 @@ def synchronize(cls, options, return_raw_response=None, timeout=3600000):
         cls.command_sub = 'synchronize'
         return cls.execute(
             cls._construct_command(options),
-            output_format='csv',
+            output_format='base',
             ignore_stderr=True,
             return_raw_response=return_raw_response,
             timeout=timeout,

robottelo/host_helpers/cli_factory.py

@@ -249,7 +249,6 @@ def __init__(self, satellite):
         self._satellite = satellite
         self.__dict__.update(initiate_repo_helpers(self._satellite))
 
-    @lru_cache
     def __getattr__(self, name):
         """We intercept the usual attribute behavior on this class to emulate make_entity methods
         The keys in the dictionary above correspond to potential make_<key> methods

robottelo/hosts.py

@@ -1856,10 +1856,10 @@ def get_caller():
         except Exception:
             raise
         finally:
-            video_url = settings.ui.grid_url.replace(
-                ':4444', f'/videos/{ui_session.ui_session_id}.mp4'
-            )
             if self.record_property is not None and settings.ui.record_video:
+                video_url = settings.ui.grid_url.replace(
+                    ':4444', f'/videos/{ui_session.ui_session_id}/video.mp4'
+                )
                 self.record_property('video_url', video_url)
                 self.record_property('session_id', ui_session.ui_session_id)
 

robottelo/utils/virtwho.py

@@ -506,7 +506,7 @@ def virtwho_package_locked():
     assert "Packages are locked" in result[1]
 
 
-def create_http_proxy(org, name=None, url=None, http_type='https'):
+def create_http_proxy(org, location, name=None, url=None, http_type='https'):
     """
     Creat a new http-proxy with attributes.
     :param name: Name of the proxy
@@ -524,6 +524,7 @@ def create_http_proxy(org, name=None, url=None, http_type='https'):
         name=http_proxy_name,
         url=http_proxy_url,
         organization=[org.id],
+        location=[location.id],
     ).create()
     return http_proxy.url, http_proxy.name, http_proxy.id
 

tests/foreman/api/test_discoveredhost.py

@@ -201,7 +201,7 @@ def test_positive_provision_pxe_host(
         mac = provisioning_host._broker_args['provisioning_nic_mac_addr']
         wait_for(
             lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [],
-            timeout=600,
+            timeout=1500,
             delay=40,
         )
         discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0]
@@ -251,7 +251,7 @@ def test_positive_provision_pxe_less_host(
         mac = pxeless_discovery_host._broker_args['provisioning_nic_mac_addr']
         wait_for(
             lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [],
-            timeout=600,
+            timeout=1500,
             delay=40,
         )
         discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0]
@@ -386,7 +386,7 @@ def test_positive_reboot_pxe_host(
         mac = provisioning_host._broker_args['provisioning_nic_mac_addr']
         wait_for(
             lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [],
-            timeout=240,
+            timeout=1500,
             delay=20,
         )
 
@@ -432,7 +432,7 @@ def test_positive_reboot_all_pxe_hosts(
             mac = host._broker_args['provisioning_nic_mac_addr']
             wait_for(
                 lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [],  # noqa: B023
-                timeout=240,
+                timeout=1500,
                 delay=20,
             )
             discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0]

tests/foreman/api/test_provisioning.py

@@ -587,6 +587,7 @@ def test_rhel_pxe_provisioning_fips_enabled(
 
 @pytest.mark.e2e
 @pytest.mark.parametrize('pxe_loader', ['bios', 'uefi'], indirect=True)
+@pytest.mark.on_premises_provisioning
 @pytest.mark.rhel_ver_match('[^6]')
 def test_capsule_pxe_provisioning(
     request,

tests/foreman/cli/test_discoveredhost.py

@@ -52,7 +52,7 @@ def test_rhel_pxe_discovery_provisioning(
 
     wait_for(
         lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [],
-        timeout=600,
+        timeout=1500,
         delay=40,
     )
     discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0]
@@ -115,7 +115,7 @@ def test_rhel_pxeless_discovery_provisioning(
 
     wait_for(
         lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [],
-        timeout=600,
+        timeout=1500,
         delay=40,
     )
     discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0]

tests/foreman/cli/test_ping.py

@@ -37,21 +37,13 @@ def test_positive_ping(target_sat, switch_user):
     """
     result = target_sat.execute(f"su - {'postgres' if switch_user else 'root'} -c 'hammer ping'")
     assert result.stderr[1].decode() == ''
-    assert result.status == 0
 
-    status_count = 0
-    ok_count = 0
-    # Exclude message from stdout for services candlepin_events and katello_events
-    result.stdout = [line for line in result.stdout.splitlines() if 'message' not in line]
+    # Filter lines containing status
+    statuses = [line for line in result.stdout.splitlines() if 'status:' in line.lower()]
 
-    # iterate over the lines grouping every 3 lines
-    # example [1, 2, 3, 4, 5, 6] will return [(1, 2, 3), (4, 5, 6)]
-    # only the status line is relevant for this test
-    for _, status, _ in zip(*[iter(result.stdout)] * 3, strict=False):  # should this be strict?
-        status_count += 1
-
-        if status.split(':')[1].strip().lower() == 'ok':
-            ok_count += 1
+    # Get count of total status lines and lines containing OK
+    status_count = len(statuses)
+    ok_count = len([status for status in statuses if status.split(':')[1].strip().lower() == 'ok'])
 
     if status_count == ok_count:
         assert result.status == 0, 'Return code should be 0 if all services are ok'

tests/foreman/destructive/test_installer.py

@@ -11,6 +11,8 @@
 :CaseImportance: Critical
 
 """
+import random
+
 from fauxfactory import gen_domain, gen_string
 import pytest
 
@@ -140,3 +142,40 @@ def test_positive_installer_certs_regenerate(target_sat):
     )
     assert result.status == 0
     assert 'FAIL' not in target_sat.cli.Base.ping()
+
+
+def test_positive_installer_puma_worker_count(target_sat):
+    """Installer should set the puma worker count and thread max without having to manually
+    restart the foreman service.
+
+    :id: d0e7d958-dd3e-4962-bf5a-8d7ec36f3485
+
+    :steps:
+        1. Check how many puma workers there are
+        2. Select a new worker count that is less than the default
+        2. Change answer's file to have new count for puma workers
+        3. Run satellite-installer --foreman-foreman-service-puma-workers new_count --foreman-foreman-service-puma-threads-max new_count
+
+    :expectedresults: aux should show there are only new_count puma workers after installer runs
+
+    :BZ: 2025760
+
+    :customerscenario: true
+    """
+    count = int(target_sat.execute('pgrep --full "puma: cluster worker" | wc -l').stdout)
+    worker_count = str(random.randint(1, count - 1))
+    result = target_sat.install(
+        InstallerCommand(
+            foreman_foreman_service_puma_workers=worker_count,
+            foreman_foreman_service_puma_threads_max=worker_count,
+        )
+    )
+    assert result.status == 0
+    result = target_sat.execute(f'grep "foreman_service_puma_workers" {SATELLITE_ANSWER_FILE}')
+    assert worker_count in result.stdout
+    result = target_sat.execute('ps aux | grep -v grep | grep -e USER -e puma')
+    for i in range(count):
+        if i < int(worker_count):
+            assert f'cluster worker {i}' in result.stdout
+        else:
+            assert f'cluster worker {i}' not in result.stdout

tests/foreman/ui/test_discoveredhost.py

@@ -81,7 +81,7 @@ def test_positive_provision_pxe_host(
     mac = provisioning_host._broker_args['provisioning_nic_mac_addr']
     wait_for(
         lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [],
-        timeout=240,
+        timeout=1500,
         delay=20,
     )
     discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0]
@@ -179,7 +179,7 @@ def test_positive_auto_provision_host_with_rule(
     mac = pxeless_discovery_host._broker_args['provisioning_nic_mac_addr']
     wait_for(
         lambda: sat.api.DiscoveredHost().search(query={'mac': mac}) != [],
-        timeout=240,
+        timeout=1500,
         delay=20,
     )
     discovered_host = sat.api.DiscoveredHost().search(query={'mac': mac})[0]

tests/foreman/virtwho/api/test_esx.py

@@ -210,7 +210,7 @@ def test_positive_filter_option(
 
     @pytest.mark.tier2
     def test_positive_proxy_option(
-        self, default_org, form_data_api, virtwho_config_api, target_sat
+        self, default_org, default_location, form_data_api, virtwho_config_api, target_sat
     ):
         """Verify http_proxy option by "PUT
 
@@ -232,7 +232,7 @@ def test_positive_proxy_option(
         assert get_configure_option('no_proxy', ETC_VIRTWHO_CONFIG) == '*'
         # Check HTTTP Proxy and No_PROXY option
         http_proxy_url, http_proxy_name, http_proxy_id = create_http_proxy(
-            http_type='http', org=default_org
+            http_type='http', org=default_org, location=default_location
         )
         no_proxy = 'test.satellite.com'
         virtwho_config_api.http_proxy_id = http_proxy_id
@@ -245,7 +245,9 @@ def test_positive_proxy_option(
         assert get_configure_option('http_proxy', ETC_VIRTWHO_CONFIG) == http_proxy_url
         assert get_configure_option('no_proxy', ETC_VIRTWHO_CONFIG) == no_proxy
         # Check HTTTPs Proxy option
-        https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy(org=default_org)
+        https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy(
+            org=default_org, location=default_location
+        )
         virtwho_config_api.http_proxy_id = https_proxy_id
         virtwho_config_api.update(['http_proxy_id'])
         deploy_configure_by_command(

tests/foreman/virtwho/api/test_esx_sca.py

@@ -227,7 +227,9 @@ def test_positive_filter_option(
                 assert result.exclude_host_parents == regex
 
     @pytest.mark.tier2
-    def test_positive_proxy_option(self, module_sca_manifest_org, form_data_api, target_sat):
+    def test_positive_proxy_option(
+        self, module_sca_manifest_org, default_location, form_data_api, target_sat
+    ):
         """Verify http_proxy option by "PUT
 
         /foreman_virt_who_configure/api/v2/configs/:id""
@@ -251,7 +253,7 @@ def test_positive_proxy_option(self, module_sca_manifest_org, form_data_api, tar
         assert get_configure_option('no_proxy', ETC_VIRTWHO_CONFIG) == '*'
         # Check HTTTP Proxy and No_PROXY option
         http_proxy_url, http_proxy_name, http_proxy_id = create_http_proxy(
-            http_type='http', org=module_sca_manifest_org
+            http_type='http', org=module_sca_manifest_org, location=default_location
         )
         no_proxy = 'test.satellite.com'
         virtwho_config.http_proxy_id = http_proxy_id
@@ -269,7 +271,7 @@ def test_positive_proxy_option(self, module_sca_manifest_org, form_data_api, tar
         assert result.no_proxy == no_proxy
         # Check HTTTPs Proxy option
         https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy(
-            org=module_sca_manifest_org
+            org=module_sca_manifest_org, location=default_location
         )
         virtwho_config.http_proxy_id = https_proxy_id
         virtwho_config.update(['http_proxy_id'])

tests/foreman/virtwho/cli/test_esx.py

@@ -214,7 +214,7 @@ def test_positive_filter_option(
 
     @pytest.mark.tier2
     def test_positive_proxy_option(
-        self, default_org, form_data_cli, virtwho_config_cli, target_sat
+        self, default_org, default_location, form_data_cli, virtwho_config_cli, target_sat
     ):
         """Verify http_proxy option by hammer virt-who-config update"
 
@@ -227,7 +227,9 @@ def test_positive_proxy_option(
         :BZ: 1902199
         """
         # Check the https proxy option, update it via http proxy name
-        https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy(org=default_org)
+        https_proxy_url, https_proxy_name, https_proxy_id = create_http_proxy(
+            org=default_org, location=default_location
+        )
         no_proxy = 'test.satellite.com'
         target_sat.cli.VirtWhoConfig.update(
             {'id': virtwho_config_cli['id'], 'http-proxy': https_proxy_name, 'no-proxy': no_proxy}
@@ -244,7 +246,7 @@ def test_positive_proxy_option(
 
         # Check the http proxy option, update it via http proxy id
         http_proxy_url, http_proxy_name, http_proxy_id = create_http_proxy(
-            http_type='http', org=default_org
+            http_type='http', org=default_org, location=default_location
         )
         target_sat.cli.VirtWhoConfig.update(
             {'id': virtwho_config_cli['id'], 'http-proxy-id': http_proxy_id}