From 99b18a8ff9794a8613de508bb0e216ea2686f141 Mon Sep 17 00:00:00 2001 From: Lukas Hellebrandt Date: Thu, 10 Oct 2024 17:18:35 +0200 Subject: [PATCH 1/9] Test template sync using proxy. --- tests/foreman/cli/test_templatesync.py | 30 +++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/foreman/cli/test_templatesync.py b/tests/foreman/cli/test_templatesync.py index 51a9305f5ed..bd04dc2b90d 100644 --- a/tests/foreman/cli/test_templatesync.py +++ b/tests/foreman/cli/test_templatesync.py @@ -52,9 +52,14 @@ def setUpClass(self, module_target_sat): f'[ -f example_template.erb ] || wget {FOREMAN_TEMPLATE_TEST_TEMPLATE}' ) + @pytest.mark.parametrize( + 'use_proxy', + [True, False], + ids=['use_proxy', 'do_not_use_proxy'], + ) @pytest.mark.tier2 def test_positive_import_force_locked_template( - self, module_org, create_import_export_local_dir, target_sat + self, module_org, create_import_export_local_dir, target_sat, use_proxy ): """Assure locked templates are updated from repository when `force` is specified. @@ -76,9 +81,12 @@ def test_positive_import_force_locked_template( """ prefix = gen_string('alpha') _, dir_path = create_import_export_local_dir + # TODO make sure the system can't communicate with the git directly, without proxy target_sat.cli.TemplateSync.imports( {'repo': dir_path, 'prefix': prefix, 'organization-ids': module_org.id, 'lock': 'true'} + # TODO specify proxy if use_proxy True ) + # TODO assert that proxy has been used ptemplate = target_sat.api.ProvisioningTemplate().search( query={'per_page': 10, 'search': f'name~{prefix}', 'organization_id': module_org.id} ) @@ -121,8 +129,13 @@ def test_positive_import_force_locked_template( indirect=True, ids=['non_empty_repo'], ) + @pytest.mark.parametrize( + 'use_proxy', + [True, False], + ids=['use_proxy', 'do_not_use_proxy'], + ) def test_positive_update_templates_in_git( - self, module_org, git_repository, git_branch, url, module_target_sat + self, module_org, git_repository, git_branch, url, module_target_sat, use_proxy ): """Assure only templates with a given filter are pushed to git repository and existing template file is updated. @@ -154,6 +167,7 @@ def test_positive_update_templates_in_git( f'{api_url}/{path}', auth=auth, json={'branch': git_branch, 'content': content} ) assert res.status_code == 201 + # TODO make sure the system can't communicate with the git directly, without proxy # export template to git url = f'{url}/{git.username}/{git_repository["name"]}' output = module_target_sat.cli.TemplateSync.exports( @@ -163,6 +177,7 @@ def test_positive_update_templates_in_git( 'organization-id': module_org.id, 'filter': 'User - Registered Users', 'dirname': dirname, + # TODO specify proxy if use_proxy True } ).split('\n') exported_count = ['Exported: true' in row.strip() for row in output].count(True) @@ -171,6 +186,7 @@ def test_positive_update_templates_in_git( git_file = requests.get(f'{api_url}/{path}', auth=auth, params={'ref': git_branch}).json() decoded = base64.b64decode(git_file['content']) assert content != decoded + # TODO assert that proxy has been used @pytest.mark.tier2 @pytest.mark.skip_if_not_set('git') @@ -188,8 +204,14 @@ def test_positive_update_templates_in_git( indirect=True, ids=['non_empty_repo', 'empty_repo'], ) + @pytest.mark.parametrize( + 'use_proxy_global', + [True, False], + ids=['use_proxy_global', 'do_not_use_proxy_global'], + ) + # TODO a settings fixture that sets using global proxy for template sync to true and after yield, resets it to the original state def test_positive_export_filtered_templates_to_git( - self, module_org, git_repository, git_branch, url, module_target_sat + self, module_org, git_repository, git_branch, url, module_target_sat, use_proxy_global ): """Assure only templates with a given filter regex are pushed to git repository. @@ -210,6 +232,7 @@ def test_positive_export_filtered_templates_to_git( """ dirname = 'export' url = f'{url}/{git.username}/{git_repository["name"]}' + # TODO make sure the system can't communicate with the git directly, without proxy output = module_target_sat.cli.TemplateSync.exports( { 'repo': url, @@ -228,6 +251,7 @@ def test_positive_export_filtered_templates_to_git( requests.get(f'{api_url}/{path}', auth=auth, params={'ref': git_branch}).json() ) assert exported_count == git_count + # TODO assert that proxy has been used @pytest.mark.tier2 def test_positive_export_filtered_templates_to_temp_dir(self, module_org, target_sat): From c0726b3a9304b07a0f2ae79cf532ba91ea32aea1 Mon Sep 17 00:00:00 2001 From: Lukas Hellebrandt Date: Wed, 23 Oct 2024 15:47:56 +0200 Subject: [PATCH 2/9] TemplateSync with proxy - API testcase --- tests/foreman/api/test_templatesync.py | 88 ++++++++++++++++++++++---- 1 file changed, 76 insertions(+), 12 deletions(-) diff --git a/tests/foreman/api/test_templatesync.py b/tests/foreman/api/test_templatesync.py index d47d6335ba7..58cbc69d5c9 100644 --- a/tests/foreman/api/test_templatesync.py +++ b/tests/foreman/api/test_templatesync.py @@ -18,6 +18,7 @@ import pytest import requests +from robottelo import ssh from robottelo.config import settings from robottelo.constants import ( FOREMAN_TEMPLATE_IMPORT_API_URL, @@ -28,6 +29,7 @@ FOREMAN_TEMPLATES_NOT_IMPORTED_COUNT, ) from robottelo.logging import logger +from robottelo.utils.issue_handlers import is_open git = settings.git @@ -1078,13 +1080,34 @@ def test_positive_export_log_to_production( indirect=True, ids=['non_empty_repo', 'empty_repo'], ) + @pytest.mark.parametrize( + 'use_proxy', + [True, False], + ids=['use_proxy', 'do_not_use_proxy'], + ) + @pytest.mark.parametrize( + 'setup_http_proxy', + [True, False], + indirect=True, + ids=['auth_http_proxy', 'unauth_http_proxy'], + ) def test_positive_export_all_templates_to_repo( - self, module_org, git_repository, git_branch, url, module_target_sat + self, + module_org, + git_repository, + git_branch, + url, + module_target_sat, + use_proxy, + setup_http_proxy, ): """Assure all templates are exported if no filter is specified. :id: 0bf6fe77-01a3-4843-86d6-22db5b8adf3b + :setup: + 1. If using proxy, disable direct connection to the git instance + :steps: 1. Using nailgun export all templates to repository (ensure filters are empty) @@ -1097,21 +1120,53 @@ def test_positive_export_all_templates_to_repo( :CaseImportance: Low """ - output = module_target_sat.api.Template().exports( - data={ + # TODO remove this + if is_open('SAT-28933') and 'ssh' in url: + pytest.skip("Temporary skip of SSH tests") + proxy, param = setup_http_proxy + if not use_proxy and not param: + # only do-not-use one kind of proxy + pytest.skip( + "Invalid parameter combination. DO NOT USE PROXY scenario should only be tested once." + ) + try: + data = { 'repo': f'{url}/{git.username}/{git_repository["name"]}', 'branch': git_branch, 'organization_ids': [module_org.id], } - ) - auth = (git.username, git.password) - api_url = f'http://{git.hostname}:{git.http_port}/api/v1/repos/{git.username}' - res = requests.get( - url=f'{api_url}/{git_repository["name"]}/git/trees/{git_branch}', - auth=auth, - params={'recursive': True}, - ) - res.raise_for_status() + if use_proxy: + proxy_hostname = setup_http_proxy[0].url.split('/')[2].split(':')[0] + log_path = '/var/log/squid/access.log' + old_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() + ssh.command( + f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {old_log}' + ) + # make sure the system can't communicate with the git directly, without proxy + assert ( + module_target_sat.execute( + f'firewall-cmd --permanent --direct --add-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' + ).status + == 0 + ) + assert module_target_sat.execute(f'ping -c 2 {settings.git.hostname}').status != 0 + data['http_proxy_policy'] = 'selected' + data['http_proxy_id'] = proxy.id + output = module_target_sat.api.Template().exports(data=data) + auth = (git.username, git.password) + api_url = f'http://{git.hostname}:{git.http_port}/api/v1/repos/{git.username}' + res = requests.get( + url=f'{api_url}/{git_repository["name"]}/git/trees/{git_branch}', + auth=auth, + params={'recursive': True}, + ) + res.raise_for_status() + finally: + if use_proxy: + module_target_sat.execute( + f'firewall-cmd --permanent --direct --remove-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' + ) + try: tree = json.loads(res.text)['tree'] except json.decoder.JSONDecodeError: @@ -1119,6 +1174,15 @@ def test_positive_export_all_templates_to_repo( pytest.fail(f"Failed to parse output from git. Response: '{res.text}'") git_count = [row['path'].endswith('.erb') for row in tree].count(True) assert len(output['message']['templates']) == git_count + # assert that proxy has been used + if use_proxy: + new_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() + ssh.command( + f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {new_log}' + ) + diff = ssh.command(f'diff {old_log} {new_log}').stdout + satellite_ip = ssh.command('dig A +short $(hostname)').stdout.strip() + assert satellite_ip in diff @pytest.mark.tier2 def test_positive_import_all_templates_from_repo(self, module_org, module_target_sat): From 62eb0550ba18702c88e36d9e10131fa84730f643 Mon Sep 17 00:00:00 2001 From: Lukas Hellebrandt Date: Tue, 29 Oct 2024 17:08:25 +0100 Subject: [PATCH 3/9] Make sure global proxy is not being used --- pytest_fixtures/component/http_proxy.py | 9 +++++++++ tests/foreman/api/test_templatesync.py | 8 ++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/pytest_fixtures/component/http_proxy.py b/pytest_fixtures/component/http_proxy.py index 6fea394f4ad..71ff19cb0b6 100644 --- a/pytest_fixtures/component/http_proxy.py +++ b/pytest_fixtures/component/http_proxy.py @@ -31,3 +31,12 @@ def setup_http_proxy(request, module_manifest_org, target_sat): target_sat.update_setting('http_proxy', general_proxy_value) if http_proxy: http_proxy.delete() + + +@pytest.fixture +def setup_http_proxy_without_global_settings(request, module_manifest_org, target_sat): + """Create a new HTTP proxy but don't set it as global or content proxy""" + http_proxy = target_sat.api_factory.make_http_proxy(module_manifest_org, request.param) + yield http_proxy, request.param + if http_proxy: + http_proxy.delete() diff --git a/tests/foreman/api/test_templatesync.py b/tests/foreman/api/test_templatesync.py index 58cbc69d5c9..2ffe39650ac 100644 --- a/tests/foreman/api/test_templatesync.py +++ b/tests/foreman/api/test_templatesync.py @@ -1086,7 +1086,7 @@ def test_positive_export_log_to_production( ids=['use_proxy', 'do_not_use_proxy'], ) @pytest.mark.parametrize( - 'setup_http_proxy', + 'setup_http_proxy_without_global_settings', [True, False], indirect=True, ids=['auth_http_proxy', 'unauth_http_proxy'], @@ -1099,7 +1099,7 @@ def test_positive_export_all_templates_to_repo( url, module_target_sat, use_proxy, - setup_http_proxy, + setup_http_proxy_without_global_settings, ): """Assure all templates are exported if no filter is specified. @@ -1123,7 +1123,7 @@ def test_positive_export_all_templates_to_repo( # TODO remove this if is_open('SAT-28933') and 'ssh' in url: pytest.skip("Temporary skip of SSH tests") - proxy, param = setup_http_proxy + proxy, param = setup_http_proxy_without_global_settings if not use_proxy and not param: # only do-not-use one kind of proxy pytest.skip( @@ -1136,7 +1136,7 @@ def test_positive_export_all_templates_to_repo( 'organization_ids': [module_org.id], } if use_proxy: - proxy_hostname = setup_http_proxy[0].url.split('/')[2].split(':')[0] + proxy_hostname = proxy.url.split('/')[2].split(':')[0] log_path = '/var/log/squid/access.log' old_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() ssh.command( From 02e64ffb2459d0c905f514423d78115072e78127 Mon Sep 17 00:00:00 2001 From: Lukas Hellebrandt Date: Fri, 1 Nov 2024 16:05:02 +0100 Subject: [PATCH 4/9] CLI, global proxy template sync --- pytest_fixtures/component/http_proxy.py | 17 +++ robottelo/cli/base.py | 4 +- tests/foreman/cli/test_templatesync.py | 139 +++++++++++++++++++----- 3 files changed, 131 insertions(+), 29 deletions(-) diff --git a/pytest_fixtures/component/http_proxy.py b/pytest_fixtures/component/http_proxy.py index 71ff19cb0b6..1e291167012 100644 --- a/pytest_fixtures/component/http_proxy.py +++ b/pytest_fixtures/component/http_proxy.py @@ -40,3 +40,20 @@ def setup_http_proxy_without_global_settings(request, module_manifest_org, targe yield http_proxy, request.param if http_proxy: http_proxy.delete() + + +@pytest.fixture +def setup_http_proxy_global(request, target_sat): + """Create a new HTTP proxy and set related settings based on proxy""" + if request.param: + hostname = settings.http_proxy.auth_proxy_url[7:] + general_proxy = ( + f'http://{settings.http_proxy.username}:' f'{settings.http_proxy.password}@{hostname}' + ) + else: + general_proxy = settings.http_proxy.un_auth_proxy_url + general_proxy_value = target_sat.update_setting( + 'http_proxy', general_proxy if request.param is not None else '' + ) + yield general_proxy, request.param + target_sat.update_setting('http_proxy', general_proxy_value) diff --git a/robottelo/cli/base.py b/robottelo/cli/base.py index e7162da396e..fbbbc122325 100644 --- a/robottelo/cli/base.py +++ b/robottelo/cli/base.py @@ -192,7 +192,9 @@ def execute( time_hammer = settings.performance.time_hammer # add time to measure hammer performance - cmd = 'LANG={} {} hammer -v {} {} {} {}'.format( + # TODO workaround, remove before commiting + # cmd = 'LANG={} {} hammer -v {} {} {} {}'.format( + cmd = 'LANG={} {} hammer {} {} {} {}'.format( settings.robottelo.locale, 'time -p' if time_hammer else '', f'-u {user}' if user else "--interactive no", diff --git a/tests/foreman/cli/test_templatesync.py b/tests/foreman/cli/test_templatesync.py index bd04dc2b90d..e231d3179d4 100644 --- a/tests/foreman/cli/test_templatesync.py +++ b/tests/foreman/cli/test_templatesync.py @@ -21,6 +21,8 @@ FOREMAN_TEMPLATE_IMPORT_URL, FOREMAN_TEMPLATE_TEST_TEMPLATE, ) +from robottelo.utils import ssh +from robottelo.utils.issue_handlers import is_open git = settings.git @@ -52,14 +54,9 @@ def setUpClass(self, module_target_sat): f'[ -f example_template.erb ] || wget {FOREMAN_TEMPLATE_TEST_TEMPLATE}' ) - @pytest.mark.parametrize( - 'use_proxy', - [True, False], - ids=['use_proxy', 'do_not_use_proxy'], - ) @pytest.mark.tier2 def test_positive_import_force_locked_template( - self, module_org, create_import_export_local_dir, target_sat, use_proxy + self, module_org, create_import_export_local_dir, target_sat ): """Assure locked templates are updated from repository when `force` is specified. @@ -81,12 +78,9 @@ def test_positive_import_force_locked_template( """ prefix = gen_string('alpha') _, dir_path = create_import_export_local_dir - # TODO make sure the system can't communicate with the git directly, without proxy target_sat.cli.TemplateSync.imports( {'repo': dir_path, 'prefix': prefix, 'organization-ids': module_org.id, 'lock': 'true'} - # TODO specify proxy if use_proxy True ) - # TODO assert that proxy has been used ptemplate = target_sat.api.ProvisioningTemplate().search( query={'per_page': 10, 'search': f'name~{prefix}', 'organization_id': module_org.id} ) @@ -112,6 +106,112 @@ def test_positive_import_force_locked_template( else: pytest.fail('The template is not imported for force test') + @pytest.mark.skip_if_not_set('git') + @pytest.mark.parametrize( + 'url', + [ + 'https://github.com/theforeman/community-templates.git', + 'ssh://git@github.com/theforeman/community-templates.git', + ], + ids=['http', 'ssh'], + ) + @pytest.mark.parametrize( + 'setup_http_proxy_global', + [True, False], + indirect=True, + ids=['auth_http_proxy_global', 'unauth_http_proxy_global'], + ) + @pytest.mark.parametrize( + 'use_proxy_global', + [True, False], + ids=['use_proxy_global', 'do_not_use_proxy_global'], + ) + @pytest.mark.tier2 + def test_positive_import_dir_filtered( + self, + module_org, + create_import_export_local_dir, + target_sat, + use_proxy_global, + setup_http_proxy_global, + url, + ): + """Import a template from git, specifying directory and filter + + :id: 17bfb25a-e215-4f57-b861-294cd018bcf1 + + :setup: + 1. Unlock and remove a template to be imported + + :steps: + 1. Import a template, specifying its dir and filter + + :expectedresults: + 1. The template is present + + :CaseImportance: Medium + """ + # TODO remove this + if is_open('SAT-28933') and 'ssh' in url: + pytest.skip("Temporary skip of SSH tests") + proxy, param = setup_http_proxy_global + if not use_proxy_global and not param: + # only do-not-use one kind of proxy + pytest.skip( + "Invalid parameter combination. DO NOT USE PROXY scenario should only be tested once." + ) + pt_name = 'FreeBSD default fake' + if target_sat.cli.PartitionTable.list({'search': f'name=\\"{pt_name}\\"'}): + target_sat.cli.PartitionTable.update({'name': pt_name, 'locked': 0}) + target_sat.cli.PartitionTable.delete({'name': pt_name}) + try: + data = { + 'repo': url, + 'organization-ids': module_org.id, + 'branch': 'develop', + 'dirname': '/partition_tables_templates/', + 'filter': pt_name, + } + if use_proxy_global: + proxy_hostname = ( + proxy.split('/')[2].split(':')[0] + if '@' not in proxy + else proxy.split('@')[1].split(':')[0] + ) + log_path = '/var/log/squid/access.log' + old_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() + ssh.command( + f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {old_log}' + ) + # make sure the system can't communicate with the git directly, without proxy + assert ( + target_sat.execute( + f'firewall-cmd --permanent --direct --add-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' + ).status + == 0 + ) + assert target_sat.execute(f'ping -c 2 {settings.git.hostname}').status != 0 + data['http-proxy-policy'] = 'global' + target_sat.cli.TemplateSync.imports(data) + finally: + if use_proxy_global: + target_sat.execute( + f'firewall-cmd --permanent --direct --remove-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' + ) + # assert that template has been synced -> is present on the Satellite + pt = target_sat.cli.PartitionTable.list({'search': f'name=\\"{pt_name}\\"'}) + assert len(pt) == 1 + assert pt_name == pt[0]['name'] + # assert that proxy has been used + if use_proxy_global: + new_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() + ssh.command( + f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {new_log}' + ) + diff = ssh.command(f'diff {old_log} {new_log}').stdout + satellite_ip = ssh.command('dig A +short $(hostname)').stdout.strip() + assert satellite_ip in diff + @pytest.mark.e2e @pytest.mark.tier2 @pytest.mark.skip_if_not_set('git') @@ -129,13 +229,8 @@ def test_positive_import_force_locked_template( indirect=True, ids=['non_empty_repo'], ) - @pytest.mark.parametrize( - 'use_proxy', - [True, False], - ids=['use_proxy', 'do_not_use_proxy'], - ) def test_positive_update_templates_in_git( - self, module_org, git_repository, git_branch, url, module_target_sat, use_proxy + self, module_org, git_repository, git_branch, url, module_target_sat ): """Assure only templates with a given filter are pushed to git repository and existing template file is updated. @@ -167,8 +262,6 @@ def test_positive_update_templates_in_git( f'{api_url}/{path}', auth=auth, json={'branch': git_branch, 'content': content} ) assert res.status_code == 201 - # TODO make sure the system can't communicate with the git directly, without proxy - # export template to git url = f'{url}/{git.username}/{git_repository["name"]}' output = module_target_sat.cli.TemplateSync.exports( { @@ -177,7 +270,6 @@ def test_positive_update_templates_in_git( 'organization-id': module_org.id, 'filter': 'User - Registered Users', 'dirname': dirname, - # TODO specify proxy if use_proxy True } ).split('\n') exported_count = ['Exported: true' in row.strip() for row in output].count(True) @@ -186,7 +278,6 @@ def test_positive_update_templates_in_git( git_file = requests.get(f'{api_url}/{path}', auth=auth, params={'ref': git_branch}).json() decoded = base64.b64decode(git_file['content']) assert content != decoded - # TODO assert that proxy has been used @pytest.mark.tier2 @pytest.mark.skip_if_not_set('git') @@ -204,14 +295,8 @@ def test_positive_update_templates_in_git( indirect=True, ids=['non_empty_repo', 'empty_repo'], ) - @pytest.mark.parametrize( - 'use_proxy_global', - [True, False], - ids=['use_proxy_global', 'do_not_use_proxy_global'], - ) - # TODO a settings fixture that sets using global proxy for template sync to true and after yield, resets it to the original state def test_positive_export_filtered_templates_to_git( - self, module_org, git_repository, git_branch, url, module_target_sat, use_proxy_global + self, module_org, git_repository, git_branch, url, module_target_sat ): """Assure only templates with a given filter regex are pushed to git repository. @@ -232,7 +317,6 @@ def test_positive_export_filtered_templates_to_git( """ dirname = 'export' url = f'{url}/{git.username}/{git_repository["name"]}' - # TODO make sure the system can't communicate with the git directly, without proxy output = module_target_sat.cli.TemplateSync.exports( { 'repo': url, @@ -251,7 +335,6 @@ def test_positive_export_filtered_templates_to_git( requests.get(f'{api_url}/{path}', auth=auth, params={'ref': git_branch}).json() ) assert exported_count == git_count - # TODO assert that proxy has been used @pytest.mark.tier2 def test_positive_export_filtered_templates_to_temp_dir(self, module_org, target_sat): From 16af3e387c864d86be90805afdb5e29f36e2abae Mon Sep 17 00:00:00 2001 From: Lukas Hellebrandt Date: Wed, 6 Nov 2024 13:33:14 +0100 Subject: [PATCH 5/9] UI part --- tests/foreman/ui/test_templatesync.py | 111 ++++++++++++++++++++------ 1 file changed, 86 insertions(+), 25 deletions(-) diff --git a/tests/foreman/ui/test_templatesync.py b/tests/foreman/ui/test_templatesync.py index e76444ee4a0..49f1d6a4c98 100644 --- a/tests/foreman/ui/test_templatesync.py +++ b/tests/foreman/ui/test_templatesync.py @@ -14,6 +14,7 @@ import pytest import requests +from robottelo import ssh from robottelo.config import settings from robottelo.constants import FOREMAN_TEMPLATE_IMPORT_URL, FOREMAN_TEMPLATE_ROOT_DIR @@ -31,9 +32,28 @@ def templates_loc(templates_org, module_target_sat): git = settings.git +@pytest.mark.skip_if_not_set('git') +@pytest.mark.parametrize( + 'setup_http_proxy_without_global_settings', + [True, False], + indirect=True, + ids=['auth_http_proxy', 'unauth_http_proxy'], +) +@pytest.mark.parametrize( + 'use_proxy', + [True, False], + ids=['use_proxy', 'do_not_use_proxy'], +) @pytest.mark.tier2 @pytest.mark.upgrade -def test_positive_import_templates(session, templates_org, templates_loc): +def test_positive_import_templates( + session, + templates_org, + templates_loc, + use_proxy, + setup_http_proxy_without_global_settings, + target_sat, +): """Import template(s) from external source to satellite :id: 524bf384-703f-48a5-95ff-7c1cf97db694 @@ -58,34 +78,75 @@ def test_positive_import_templates(session, templates_org, templates_loc): :CaseImportance: Critical """ + proxy, param = setup_http_proxy_without_global_settings + if not use_proxy and not param: + # only do-not-use one kind of proxy + pytest.skip( + "Invalid parameter combination. DO NOT USE PROXY scenario should only be tested once." + ) import_template = 'Alterator default PXELinux' branch = 'automation' prefix_name = gen_string('alpha', 8) - with session: - session.organization.select(org_name=templates_org.name) - session.location.select(loc_name=templates_loc.name) - import_title = session.sync_template.sync( - { - 'sync_type': 'Import', - 'template.associate': 'Always', - 'template.branch': branch, - 'template.dirname': 'provisioning_templates', - 'template.filter': import_template, - 'template.lock': 'Lock', - 'template.prefix': f'{prefix_name} ', - 'template.repo': FOREMAN_TEMPLATE_IMPORT_URL, - } + # put the http proxy in the correct org and loc + target_sat.cli.HttpProxy.update( + {'id': proxy.id, 'organization-ids': [templates_org.id], 'location-ids': [templates_loc.id]} + ) + try: + data = { + 'sync_type': 'Import', + 'template.associate': 'Always', + 'template.branch': branch, + 'template.dirname': 'provisioning_templates', + 'template.filter': import_template, + 'template.lock': 'Lock', + 'template.prefix': f'{prefix_name} ', + 'template.repo': FOREMAN_TEMPLATE_IMPORT_URL, + } + if use_proxy: + proxy_hostname = proxy.url.split('/')[2].split(':')[0] + log_path = '/var/log/squid/access.log' + old_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() + ssh.command( + f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {old_log}' + ) + # make sure the system can't communicate with the git directly, without proxy + assert ( + target_sat.execute( + f'firewall-cmd --permanent --direct --add-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' + ).status + == 0 + ) + assert target_sat.execute(f'ping -c 2 {settings.git.hostname}').status != 0 + data['template.http_proxy_policy'] = 'Use selected HTTP proxy' + data['template.http_proxy_id'] = proxy.name + with session: + session.organization.select(org_name=templates_org.name) + session.location.select(loc_name=templates_loc.name) + import_title = session.sync_template.sync(data) + assert import_title == f'Import from {FOREMAN_TEMPLATE_IMPORT_URL} and branch {branch}' + imported_template = f'{prefix_name} {import_template}' + assert session.provisioningtemplate.is_locked(imported_template) + pt = session.provisioningtemplate.read(imported_template) + finally: + if use_proxy: + target_sat.execute( + f'firewall-cmd --permanent --direct --remove-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' + ) + assert pt['template']['name'] == imported_template + assert pt['template']['default'] is False + assert pt['type']['snippet'] is False + assert pt['locations']['resources']['assigned'][0] == templates_loc.name + assert pt['organizations']['resources']['assigned'][0] == templates_org.name + assert f'name: {import_template}' in pt['template']['template_editor']['editor'] + # assert that proxy has been used + if use_proxy: + new_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() + ssh.command( + f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {new_log}' ) - assert import_title == f'Import from {FOREMAN_TEMPLATE_IMPORT_URL} and branch {branch}' - imported_template = f'{prefix_name} {import_template}' - assert session.provisioningtemplate.is_locked(imported_template) - pt = session.provisioningtemplate.read(imported_template) - assert pt['template']['name'] == imported_template - assert pt['template']['default'] is False - assert pt['type']['snippet'] is False - assert pt['locations']['resources']['assigned'][0] == templates_loc.name - assert pt['organizations']['resources']['assigned'][0] == templates_org.name - assert f'name: {import_template}' in pt['template']['template_editor']['editor'] + diff = ssh.command(f'diff {old_log} {new_log}').stdout + satellite_ip = ssh.command('dig A +short $(hostname)').stdout.strip() + assert satellite_ip in diff @pytest.mark.tier2 From b0e632821497d16f215227c5e9c6c86112775700 Mon Sep 17 00:00:00 2001 From: Lukas Hellebrandt Date: Wed, 6 Nov 2024 23:20:45 +0100 Subject: [PATCH 6/9] Improve structure - WebUI --- robottelo/host_helpers/capsule_mixins.py | 34 ++++++++++++++++++++++++ tests/foreman/ui/test_templatesync.py | 28 ++----------------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/robottelo/host_helpers/capsule_mixins.py b/robottelo/host_helpers/capsule_mixins.py index 2073df02fdf..43120e84149 100644 --- a/robottelo/host_helpers/capsule_mixins.py +++ b/robottelo/host_helpers/capsule_mixins.py @@ -4,6 +4,8 @@ from box import Box from dateutil.parser import parse +from robottelo import ssh +from robottelo.config import settings from robottelo.constants import ( PULP_ARTIFACT_DIR, PUPPET_CAPSULE_INSTALLER, @@ -189,3 +191,35 @@ def get_artifact_info(self, checksum=None, path=None): info = self.execute(f'file {path}').stdout.strip().split(': ')[1] return Box(path=path, size=size, sum=real_sum, info=info) + + def cutoff_host_setup_log(self, proxy_hostname, hostname): + """For testing of HTTP Proxy, disable direct connection to some host using firewall. On the Proxy, setup logs for later comparison that the Proxy was used.""" + log_path = '/var/log/squid/access.log' + old_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() + ssh.command( + f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {old_log}' + ) + # make sure the system can't communicate with the git directly, without proxy + assert ( + self.execute( + f'firewall-cmd --permanent --direct --add-rule ipv4 filter OUTPUT 1 -d $(dig +short A {hostname}) -j REJECT && firewall-cmd --reload' + ).status + == 0 + ) + assert self.execute(f'ping -c 2 {hostname}').status != 0 + return old_log + + def restore_host_check_log(self, proxy_hostname, hostname, old_log): + """For testing of HTTP Proxy, call after running the thing that should use Proxy.""" + log_path = '/var/log/squid/access.log' + self.execute( + f'firewall-cmd --permanent --direct --remove-rule ipv4 filter OUTPUT 1 -d $(dig +short A {hostname}) -j REJECT && firewall-cmd --reload' + ) + new_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() + ssh.command( + f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {new_log}' + ) + diff = ssh.command(f'diff {old_log} {new_log}').stdout + satellite_ip = ssh.command('dig A +short $(hostname)').stdout.strip() + # assert that proxy has been used + assert satellite_ip in diff diff --git a/tests/foreman/ui/test_templatesync.py b/tests/foreman/ui/test_templatesync.py index 49f1d6a4c98..080cbd29b6d 100644 --- a/tests/foreman/ui/test_templatesync.py +++ b/tests/foreman/ui/test_templatesync.py @@ -14,7 +14,6 @@ import pytest import requests -from robottelo import ssh from robottelo.config import settings from robottelo.constants import FOREMAN_TEMPLATE_IMPORT_URL, FOREMAN_TEMPLATE_ROOT_DIR @@ -104,19 +103,7 @@ def test_positive_import_templates( } if use_proxy: proxy_hostname = proxy.url.split('/')[2].split(':')[0] - log_path = '/var/log/squid/access.log' - old_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() - ssh.command( - f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {old_log}' - ) - # make sure the system can't communicate with the git directly, without proxy - assert ( - target_sat.execute( - f'firewall-cmd --permanent --direct --add-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' - ).status - == 0 - ) - assert target_sat.execute(f'ping -c 2 {settings.git.hostname}').status != 0 + old_log = target_sat.cutoff_host_setup_log(proxy_hostname, settings.git.hostname) data['template.http_proxy_policy'] = 'Use selected HTTP proxy' data['template.http_proxy_id'] = proxy.name with session: @@ -129,24 +116,13 @@ def test_positive_import_templates( pt = session.provisioningtemplate.read(imported_template) finally: if use_proxy: - target_sat.execute( - f'firewall-cmd --permanent --direct --remove-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' - ) + target_sat.restore_host_check_log(proxy_hostname, settings.git.hostname, old_log) assert pt['template']['name'] == imported_template assert pt['template']['default'] is False assert pt['type']['snippet'] is False assert pt['locations']['resources']['assigned'][0] == templates_loc.name assert pt['organizations']['resources']['assigned'][0] == templates_org.name assert f'name: {import_template}' in pt['template']['template_editor']['editor'] - # assert that proxy has been used - if use_proxy: - new_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() - ssh.command( - f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {new_log}' - ) - diff = ssh.command(f'diff {old_log} {new_log}').stdout - satellite_ip = ssh.command('dig A +short $(hostname)').stdout.strip() - assert satellite_ip in diff @pytest.mark.tier2 From f090398036242a5b4618293825dc5e072c70ea3d Mon Sep 17 00:00:00 2001 From: Lukas Hellebrandt Date: Thu, 7 Nov 2024 00:15:01 +0100 Subject: [PATCH 7/9] Improve structure - CLI, API --- tests/foreman/api/test_templatesync.py | 28 ++++---------------------- tests/foreman/cli/test_templatesync.py | 28 ++------------------------ 2 files changed, 6 insertions(+), 50 deletions(-) diff --git a/tests/foreman/api/test_templatesync.py b/tests/foreman/api/test_templatesync.py index 2ffe39650ac..c9d86bf17c7 100644 --- a/tests/foreman/api/test_templatesync.py +++ b/tests/foreman/api/test_templatesync.py @@ -18,7 +18,6 @@ import pytest import requests -from robottelo import ssh from robottelo.config import settings from robottelo.constants import ( FOREMAN_TEMPLATE_IMPORT_API_URL, @@ -1137,19 +1136,9 @@ def test_positive_export_all_templates_to_repo( } if use_proxy: proxy_hostname = proxy.url.split('/')[2].split(':')[0] - log_path = '/var/log/squid/access.log' - old_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() - ssh.command( - f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {old_log}' + old_log = module_target_sat.cutoff_host_setup_log( + proxy_hostname, settings.git.hostname ) - # make sure the system can't communicate with the git directly, without proxy - assert ( - module_target_sat.execute( - f'firewall-cmd --permanent --direct --add-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' - ).status - == 0 - ) - assert module_target_sat.execute(f'ping -c 2 {settings.git.hostname}').status != 0 data['http_proxy_policy'] = 'selected' data['http_proxy_id'] = proxy.id output = module_target_sat.api.Template().exports(data=data) @@ -1163,8 +1152,8 @@ def test_positive_export_all_templates_to_repo( res.raise_for_status() finally: if use_proxy: - module_target_sat.execute( - f'firewall-cmd --permanent --direct --remove-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' + module_target_sat.restore_host_check_log( + proxy_hostname, settings.git.hostname, old_log ) try: @@ -1174,15 +1163,6 @@ def test_positive_export_all_templates_to_repo( pytest.fail(f"Failed to parse output from git. Response: '{res.text}'") git_count = [row['path'].endswith('.erb') for row in tree].count(True) assert len(output['message']['templates']) == git_count - # assert that proxy has been used - if use_proxy: - new_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() - ssh.command( - f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {new_log}' - ) - diff = ssh.command(f'diff {old_log} {new_log}').stdout - satellite_ip = ssh.command('dig A +short $(hostname)').stdout.strip() - assert satellite_ip in diff @pytest.mark.tier2 def test_positive_import_all_templates_from_repo(self, module_org, module_target_sat): diff --git a/tests/foreman/cli/test_templatesync.py b/tests/foreman/cli/test_templatesync.py index e231d3179d4..1b67bf60c81 100644 --- a/tests/foreman/cli/test_templatesync.py +++ b/tests/foreman/cli/test_templatesync.py @@ -21,7 +21,6 @@ FOREMAN_TEMPLATE_IMPORT_URL, FOREMAN_TEMPLATE_TEST_TEMPLATE, ) -from robottelo.utils import ssh from robottelo.utils.issue_handlers import is_open git = settings.git @@ -178,39 +177,16 @@ def test_positive_import_dir_filtered( if '@' not in proxy else proxy.split('@')[1].split(':')[0] ) - log_path = '/var/log/squid/access.log' - old_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() - ssh.command( - f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {old_log}' - ) - # make sure the system can't communicate with the git directly, without proxy - assert ( - target_sat.execute( - f'firewall-cmd --permanent --direct --add-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' - ).status - == 0 - ) - assert target_sat.execute(f'ping -c 2 {settings.git.hostname}').status != 0 + old_log = target_sat.cutoff_host_setup_log(proxy_hostname, settings.git.hostname) data['http-proxy-policy'] = 'global' target_sat.cli.TemplateSync.imports(data) finally: if use_proxy_global: - target_sat.execute( - f'firewall-cmd --permanent --direct --remove-rule ipv4 filter OUTPUT 1 -d $(dig +short A {settings.git.hostname}) -j REJECT && firewall-cmd --reload' - ) + target_sat.restore_host_check_log(proxy_hostname, settings.git.hostname, old_log) # assert that template has been synced -> is present on the Satellite pt = target_sat.cli.PartitionTable.list({'search': f'name=\\"{pt_name}\\"'}) assert len(pt) == 1 assert pt_name == pt[0]['name'] - # assert that proxy has been used - if use_proxy_global: - new_log = ssh.command('echo /tmp/$RANDOM').stdout.strip() - ssh.command( - f'sshpass -p "{settings.server.ssh_password}" scp -o StrictHostKeyChecking=no root@{proxy_hostname}:{log_path} {new_log}' - ) - diff = ssh.command(f'diff {old_log} {new_log}').stdout - satellite_ip = ssh.command('dig A +short $(hostname)').stdout.strip() - assert satellite_ip in diff @pytest.mark.e2e @pytest.mark.tier2 From ea4fc79623bf4547c8d24e3d99e223950e9b6e3f Mon Sep 17 00:00:00 2001 From: Lukas Hellebrandt Date: Mon, 11 Nov 2024 17:04:08 +0100 Subject: [PATCH 8/9] Remove workaround of SAT-29212 --- robottelo/cli/base.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/robottelo/cli/base.py b/robottelo/cli/base.py index fbbbc122325..e7162da396e 100644 --- a/robottelo/cli/base.py +++ b/robottelo/cli/base.py @@ -192,9 +192,7 @@ def execute( time_hammer = settings.performance.time_hammer # add time to measure hammer performance - # TODO workaround, remove before commiting - # cmd = 'LANG={} {} hammer -v {} {} {} {}'.format( - cmd = 'LANG={} {} hammer {} {} {} {}'.format( + cmd = 'LANG={} {} hammer -v {} {} {} {}'.format( settings.robottelo.locale, 'time -p' if time_hammer else '', f'-u {user}' if user else "--interactive no", From 66ae35b95304524308933f692969a1df80205af6 Mon Sep 17 00:00:00 2001 From: Lukas Hellebrandt Date: Thu, 12 Dec 2024 15:27:49 +0100 Subject: [PATCH 9/9] Update label --- tests/foreman/ui/test_templatesync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/foreman/ui/test_templatesync.py b/tests/foreman/ui/test_templatesync.py index 080cbd29b6d..d55e43d51d4 100644 --- a/tests/foreman/ui/test_templatesync.py +++ b/tests/foreman/ui/test_templatesync.py @@ -104,7 +104,7 @@ def test_positive_import_templates( if use_proxy: proxy_hostname = proxy.url.split('/')[2].split(':')[0] old_log = target_sat.cutoff_host_setup_log(proxy_hostname, settings.git.hostname) - data['template.http_proxy_policy'] = 'Use selected HTTP proxy' + data['template.http_proxy_policy'] = 'Custom HTTP proxy' data['template.http_proxy_id'] = proxy.name with session: session.organization.select(org_name=templates_org.name)