diff --git a/robottelo/cli/capsule.py b/robottelo/cli/capsule.py index c5ba0ec8918..71fa3d9f81d 100644 --- a/robottelo/cli/capsule.py +++ b/robottelo/cli/capsule.py @@ -100,6 +100,14 @@ def content_update_counts(cls, options): return cls.execute(cls._construct_command(options), output_format='json') + @classmethod + def content_verify_checksum(cls, options): + """Trigger verify checksum task.""" + + cls.command_sub = 'content verify-checksum' + + return cls.execute(cls._construct_command(options), output_format='json') + @classmethod def import_classes(cls, options): """Import puppet classes from puppet Capsule.""" diff --git a/robottelo/constants/__init__.py b/robottelo/constants/__init__.py index 2df7c554225..af477570073 100644 --- a/robottelo/constants/__init__.py +++ b/robottelo/constants/__init__.py @@ -714,7 +714,6 @@ CUSTOM_LOCAL_FILE = '/var/lib/pulp/imports/myrepo/test.txt' CUSTOM_FILE_REPO_FILES_COUNT = 3 CUSTOM_RPM_SHA_512_FEED_COUNT = {'rpm': 35, 'errata': 4} -CUSTOM_REPODATA_PATH = '/var/lib/pulp/published/yum/https/repos' CERT_PATH = "/etc/pki/ca-trust/source/anchors/" CERT_DATA = { 'capsule_hostname': 'capsule.example.com', @@ -859,6 +858,7 @@ } CUSTOM_PUPPET_MODULE_REPOS_VERSION = '-0.2.0.tar.gz' +PULP_ARTIFACT_DIR = '/var/lib/pulp/media/artifact/' PULP_EXPORT_DIR = '/var/lib/pulp/exports/' PULP_IMPORT_DIR = '/var/lib/pulp/imports/' EXPORT_LIBRARY_NAME = 'Export-Library' diff --git a/robottelo/host_helpers/capsule_mixins.py b/robottelo/host_helpers/capsule_mixins.py index 0e7ffd21065..4a75e095c41 100644 --- a/robottelo/host_helpers/capsule_mixins.py +++ b/robottelo/host_helpers/capsule_mixins.py @@ -1,9 +1,14 @@ from datetime import datetime, timedelta import time +from box import Box from dateutil.parser import parse -from robottelo.constants import PUPPET_CAPSULE_INSTALLER, PUPPET_COMMON_INSTALLER_OPTS +from robottelo.constants import ( + PULP_ARTIFACT_DIR, + PUPPET_CAPSULE_INSTALLER, + PUPPET_COMMON_INSTALLER_OPTS, +) from robottelo.logging import logger from robottelo.utils.installer import InstallerCommand @@ -149,3 +154,26 @@ def get_published_repo_url(self, org, prod, repo, lce=None, cv=None): if lce and cv: return f'{self.url}/pulp/content/{org}/{lce}/{cv}/custom/{prod}/{repo}/' return f'{self.url}/pulp/content/{org}/Library/custom/{prod}/{repo}/' + + def get_artifact_info(self, checksum=None, path=None): + """Returns information about pulp artifact if found on FS, + throws FileNotFoundError otherwise. + + :param checksum: Checksum of the artifact to look for. + :param path: Path to the artifact. + :return: A Box with artifact path and info. + """ + if not (checksum or path): + raise ValueError('Either checksum or path must be specified') + + if not path: + path = f'{PULP_ARTIFACT_DIR}{checksum[0:2]}/{checksum[2:]}' + + res = self.execute(f'stat --format %s {path}') + if res.status: + raise FileNotFoundError(f'Artifact not found: {path}') + size = int(res.stdout) + real_sum = self.execute(f'sha256sum {path}').stdout.split()[0] + info = self.execute(f'file {path}').stdout.strip().split(': ')[1] + + return Box(path=path, size=size, sum=real_sum, info=info) diff --git a/robottelo/host_helpers/satellite_mixins.py b/robottelo/host_helpers/satellite_mixins.py index 58fe5b5d237..a91b0a02afa 100644 --- a/robottelo/host_helpers/satellite_mixins.py +++ b/robottelo/host_helpers/satellite_mixins.py @@ -116,13 +116,15 @@ def get_repomd_revision(self, repo_url): return match.group(0) - def md5_by_url(self, url): - """Returns md5 checksum of a file, accessible via URL. Useful when you want + def checksum_by_url(self, url, sum_type='md5sum'): + """Returns desired checksum of a file, accessible via URL. Useful when you want to calculate checksum but don't want to deal with storing a file and removing it afterwards. :param str url: URL of a file. - :return str: string containing md5 checksum. + :param str sum_type: Checksum type like md5sum, sha256sum, sha512sum, etc. + Defaults to md5sum. + :return str: string containing the checksum. :raises: AssertionError: If non-zero return code received (file couldn't be reached or calculation was not successful). """ @@ -131,8 +133,8 @@ def md5_by_url(self, url): if result.status != 0: raise AssertionError(f'Failed to get `{filename}` from `{url}`.') return self.execute( - f'wget -qO - {url} | tee {filename} | md5sum | awk \'{{print $1}}\'' - ).stdout + f'wget -qO - {url} | tee {filename} | {sum_type} | awk \'{{print $1}}\'' + ).stdout.strip() def upload_manifest(self, org_id, manifest=None, interface='API', timeout=None): """Upload a manifest using the requested interface. diff --git a/tests/foreman/api/test_capsulecontent.py b/tests/foreman/api/test_capsulecontent.py index 818a808a4c2..b6a411ed45b 100644 --- a/tests/foreman/api/test_capsulecontent.py +++ b/tests/foreman/api/test_capsulecontent.py @@ -37,6 +37,7 @@ FAKE_FILE_NEW_NAME, KICKSTART_CONTENT, PRDS, + PULP_ARTIFACT_DIR, REPOS, REPOSET, RH_CONTAINER_REGISTRY_HUB, @@ -635,9 +636,9 @@ def test_positive_on_demand_sync( assert len(caps_files) == packages_count # Download a package from the Capsule and get its md5 checksum - published_package_md5 = target_sat.md5_by_url(f'{caps_repo_url}/{package}') + published_package_md5 = target_sat.checksum_by_url(f'{caps_repo_url}/{package}') # Get md5 checksum of source package - package_md5 = target_sat.md5_by_url(f'{repo_url}/{package}') + package_md5 = target_sat.checksum_by_url(f'{repo_url}/{package}') # Assert checksums are matching assert package_md5 == published_package_md5 @@ -847,8 +848,10 @@ def test_positive_sync_kickstart_repo( # Check kickstart specific files for file in KICKSTART_CONTENT: - sat_file = target_sat.md5_by_url(f'{target_sat.url}/{url_base}/{file}') - caps_file = target_sat.md5_by_url(f'{module_capsule_configured.url}/{url_base}/{file}') + sat_file = target_sat.checksum_by_url(f'{target_sat.url}/{url_base}/{file}') + caps_file = target_sat.checksum_by_url( + f'{module_capsule_configured.url}/{url_base}/{file}' + ) assert sat_file == caps_file # Check packages @@ -1162,8 +1165,8 @@ def test_positive_sync_file_repo( assert sat_files == caps_files for file in sat_files: - sat_file = target_sat.md5_by_url(f'{sat_repo_url}{file}') - caps_file = target_sat.md5_by_url(f'{caps_repo_url}{file}') + sat_file = target_sat.checksum_by_url(f'{sat_repo_url}{file}') + caps_file = target_sat.checksum_by_url(f'{caps_repo_url}{file}') assert sat_file == caps_file @pytest.mark.tier4 @@ -1370,9 +1373,7 @@ def test_positive_remove_capsule_orphans( assert sync_status['result'] == 'success', 'Capsule sync task failed.' # Ensure the RPM artifacts were created. - result = capsule_configured.execute( - 'ls /var/lib/pulp/media/artifact/*/* | xargs file | grep RPM' - ) + result = capsule_configured.execute(f'ls {PULP_ARTIFACT_DIR}*/* | xargs file | grep RPM') assert not result.status, 'RPM artifacts are missing after capsule sync.' # Remove the Library LCE from the capsule and resync it. @@ -1401,9 +1402,7 @@ def test_positive_remove_capsule_orphans( ) # Ensure the artifacts were removed. - result = capsule_configured.execute( - 'ls /var/lib/pulp/media/artifact/*/* | xargs file | grep RPM' - ) + result = capsule_configured.execute(f'ls {PULP_ARTIFACT_DIR}*/* | xargs file | grep RPM') assert result.status, 'RPM artifacts are still present. They should be gone.' @pytest.mark.skip_if_not_set('capsule') diff --git a/tests/foreman/api/test_repository.py b/tests/foreman/api/test_repository.py index e9d6271f0de..5c946a5bc53 100644 --- a/tests/foreman/api/test_repository.py +++ b/tests/foreman/api/test_repository.py @@ -1250,12 +1250,12 @@ def test_positive_sync_with_treeinfo_ignore( repo = repo.update(['mirroring_policy', 'ignorable_content']) repo.sync() with pytest.raises(AssertionError): - target_sat.md5_by_url(f'{repo.full_path}.treeinfo') + target_sat.checksum_by_url(f'{repo.full_path}.treeinfo') repo.ignorable_content = [] repo = repo.update(['ignorable_content']) repo.sync() - assert target_sat.md5_by_url( + assert target_sat.checksum_by_url( f'{repo.full_path}.treeinfo' ), 'The treeinfo file is missing in the KS repo but it should be there.' diff --git a/tests/foreman/cli/test_capsulecontent.py b/tests/foreman/cli/test_capsulecontent.py index a75ffbb1aed..c0927d7a676 100644 --- a/tests/foreman/cli/test_capsulecontent.py +++ b/tests/foreman/cli/test_capsulecontent.py @@ -11,15 +11,98 @@ :CaseImportance: High """ +from datetime import datetime +import random +from box import Box import pytest from robottelo.config import settings from robottelo.constants import ( CONTAINER_REGISTRY_HUB, CONTAINER_UPSTREAM_NAME, + PULP_ARTIFACT_DIR, ) from robottelo.constants.repos import ANSIBLE_GALAXY, CUSTOM_FILE_REPO +from robottelo.content_info import get_repo_files_urls_by_url + + +@pytest.fixture(scope='module') +def module_synced_content( + request, + module_target_sat, + module_capsule_configured, + module_org, + module_product, + module_lce, + module_lce_library, +): + """ + Create and sync one or more repositories, publish them in a CV, + promote to an LCE and sync all of that to an external Capsule. + + :param request: Repo(s) to use - dict or list of dicts with options to create the repo(s). + :return: Box with created instances and Capsule sync time. + """ + repos_opts = request.param + if not isinstance(repos_opts, list): + repos_opts = [repos_opts] + + repos = [] + for options in repos_opts: + repo = module_target_sat.api.Repository(product=module_product, **options).create() + repo.sync() + repos.append(repo) + + cv = module_target_sat.api.ContentView(organization=module_org, repository=repos).create() + cv.publish() + cvv = cv.read().version[0].read() + cvv.promote(data={'environment_ids': module_lce.id}) + + # Assign the Capsule with the LCE (if not assigned yet) and sync it. + if module_lce.id not in [ + lce['id'] for lce in module_capsule_configured.nailgun_capsule.lifecycle_environments + ]: + module_capsule_configured.nailgun_capsule.content_add_lifecycle_environment( + data={'environment_id': [module_lce.id, module_lce_library.id]} + ) + sync_time = datetime.utcnow().replace(microsecond=0) + module_target_sat.cli.Capsule.content_synchronize( + {'id': module_capsule_configured.nailgun_capsule.id, 'organization-id': module_org.id} + ) + module_capsule_configured.wait_for_sync(start_time=sync_time) + + return Box(prod=module_product, repos=repos, cv=cv, lce=module_lce, sync_time=sync_time) + + +@pytest.fixture(scope='module') +def module_capsule_artifact_cleanup( + request, + module_target_sat, + module_capsule_configured, +): + """Unassign all LCEs from the module_capsule_configured and trigger orphan cleanup task. + This should remove all pulp artifacts from the Capsule. + """ + # Remove all LCEs from the capsule + for lce in module_capsule_configured.nailgun_capsule.lifecycle_environments: + module_capsule_configured.nailgun_capsule.content_delete_lifecycle_environment( + data={'environment_id': lce['id']} + ) + # Run orphan cleanup for the capsule. + timestamp = datetime.utcnow().replace(microsecond=0) + module_target_sat.execute( + 'foreman-rake katello:delete_orphaned_content RAILS_ENV=production ' + f'SMART_PROXY_ID={module_capsule_configured.nailgun_capsule.id}' + ) + module_target_sat.wait_for_tasks( + search_query=( + 'label = Actions::Katello::OrphanCleanup::RemoveOrphans' + f' and started_at >= "{timestamp}"' + ), + search_rate=5, + max_tries=10, + ) @pytest.mark.parametrize( @@ -44,7 +127,6 @@ ], indirect=True, ) -@pytest.mark.stream def test_positive_content_counts_for_mixed_cv( target_sat, module_capsule_configured, @@ -180,7 +262,6 @@ def test_positive_content_counts_for_mixed_cv( assert len(info['lifecycle-environments']) == 0, 'The LCE is still listed' -@pytest.mark.stream def test_positive_update_counts(target_sat, module_capsule_configured): """Verify the update counts functionality @@ -205,3 +286,198 @@ def test_positive_update_counts(target_sat, module_capsule_configured): search_rate=5, max_tries=5, ) + + +@pytest.mark.stream +@pytest.mark.parametrize('repair_type', ['repo', 'cv', 'lce']) +@pytest.mark.parametrize( + 'module_synced_content', + [ + [ + {'content_type': 'yum', 'url': settings.repos.yum_0.url}, + {'content_type': 'file', 'url': CUSTOM_FILE_REPO}, + ] + ], + indirect=True, + ids=['content'], +) +@pytest.mark.parametrize('content_type', ['yum', 'file']) +@pytest.mark.parametrize('damage_type', ['destroy', 'corrupt']) +def test_positive_repair_yum_file_artifacts( + module_target_sat, + module_capsule_configured, + module_capsule_artifact_cleanup, + module_org, + module_synced_content, + damage_type, + repair_type, + content_type, +): + """Test the verify-checksum task repairs particular RPM and FILE artifacts correctly + at the Capsule side using one of its methods when they were removed or corrupted before. + + :id: f818f537-94b0-4d14-adf1-643ead828ade + + :parametrized: yes + + :setup: + 1. Have a Satellite with registered external Capsule. + 2. Clean up all previously synced artifacts from the Capsule. + 3. Create yum and file type repository, publish it in a CVV and promote to an LCE. + 4. Assign the Capsule with the LCE and sync it. + + :steps: + 1. Pick one of the published files. + 2. Cause desired type of damage to his artifact and verify the effect. + 3. Trigger desired variant of repair (verify_checksum) task. + 4. Check if the artifact is back in shape. + + :expectedresults: + 1. Artifact is stored correctly based on the checksum. + 2. All variants of verify_checksum task are able to repair all types of damage. + + :BZ: 2127537 + + :customerscenario: true + + """ + repo = next(repo for repo in module_synced_content.repos if repo.content_type == content_type) + + # Pick one of the published files. + caps_repo_url = module_capsule_configured.get_published_repo_url( + org=module_org.label, + lce=None if repair_type == 'repo' else module_synced_content.lce.label, + cv=None if repair_type == 'repo' else module_synced_content.cv.label, + prod=module_synced_content.prod.label, + repo=repo.label, + ) + cap_files_urls = get_repo_files_urls_by_url( + caps_repo_url, extension='rpm' if content_type == 'yum' else 'iso' + ) + + file_url = random.choice(cap_files_urls) + file_sum = module_target_sat.checksum_by_url(file_url, sum_type='sha256sum') + file_ai = module_capsule_configured.get_artifact_info(checksum=file_sum) + + # Cause desired type of damage to his artifact and verify the effect. + if damage_type == 'destroy': + module_capsule_configured.execute(f'rm -f {file_ai.path}') + with pytest.raises(FileNotFoundError): + module_capsule_configured.get_artifact_info(checksum=file_sum) + with pytest.raises(AssertionError): + module_target_sat.checksum_by_url(file_url) + elif damage_type == 'corrupt': + res = module_capsule_configured.execute( + f'truncate -s {random.randint(1, file_ai.size)} {file_ai.path}' + ) + assert res.status == 0, f'Artifact truncation failed: {res.stderr}' + assert ( + module_capsule_configured.get_artifact_info(checksum=file_sum) != file_ai + ), 'Artifact corruption failed' + else: + raise ValueError(f'Unsupported damage type: {damage_type}') + + # Trigger desired variant of repair (verify_checksum) task. + opts = {'id': module_capsule_configured.nailgun_capsule.id} + if repair_type == 'repo': + opts.update({'repository-id': repo.id}) + elif repair_type == 'cv': + opts.update({'content-view-id': module_synced_content.cv.id}) + elif repair_type == 'lce': + opts.update({'lifecycle-environment-id': module_synced_content.lce.id}) + module_target_sat.cli.Capsule.content_verify_checksum(opts) + + # Check if the artifact is back in shape. + fixed_ai = module_capsule_configured.get_artifact_info(checksum=file_sum) + assert fixed_ai == file_ai, f'Artifact restoration failed: {fixed_ai} != {file_ai}' + + +@pytest.mark.parametrize('repair_type', ['repo', 'cv', 'lce']) +@pytest.mark.parametrize( + 'module_synced_content', + [ + { + 'content_type': 'docker', + 'docker_upstream_name': CONTAINER_UPSTREAM_NAME, + 'url': CONTAINER_REGISTRY_HUB, + }, + { + 'content_type': 'ansible_collection', + 'url': ANSIBLE_GALAXY, + 'ansible_collection_requirements': '{collections: [ \ + { name: theforeman.foreman, version: "2.1.0" }, \ + { name: theforeman.operations, version: "0.1.0"} ]}', + }, + ], + indirect=True, + ids=['docker', 'AC'], +) +@pytest.mark.parametrize('damage_type', ['destroy', 'corrupt']) +def test_positive_repair_docker_AC_artifacts( + module_target_sat, module_capsule_configured, module_synced_content, damage_type, repair_type +): + """Test the verify-checksum task repairs particular docker and ansible-collection artifacts + correctly at the Capsule side using one of its methods when they were removed or corrupted + before. + + :id: b0e1a163-bf30-48bf-8d27-68c689ee0896 + + :parametrized: yes + + :setup: + 1. Have a Satellite with registered external Capsule. + 2. Create docker and ansible-collection type repository, publish it in a CVV and promote + to an LCE. + 3. Assign the Capsule with the LCE and sync it. + + :steps: + 1. Get all artifacts synced recently by the `module_synced_content` fixture. + 2. Pick one artifact and cause desired type of damage and verify the effect. + 3. Trigger desired variant of repair (verify_checksum) task. + 4. Check if the artifact is back in shape. + + :expectedresults: + 1. Artifact is stored correctly based on the checksum. + 2. All variants of verify_checksum task are able to repair all types of damage. + + :BZ: 2127537 + + :customerscenario: true + + """ + # Get all artifacts synced recently by the `module_synced_content` fixture. + artifacts = module_capsule_configured.execute( + f'find {PULP_ARTIFACT_DIR} -type f -newermt "{module_synced_content.sync_time} UTC"' + ).stdout.splitlines() + assert len(artifacts) > 0, 'No NEW artifacts found' + + # Pick one artifact and cause desired type of damage and verify the effect. + ai = module_capsule_configured.get_artifact_info(path=random.choice(artifacts)) + if damage_type == 'destroy': + module_capsule_configured.execute(f'rm -f {ai.path}') + with pytest.raises(FileNotFoundError): + module_capsule_configured.get_artifact_info(path=ai.path) + elif damage_type == 'corrupt': + res = module_capsule_configured.execute( + f'truncate -s {random.randint(1, ai.size)} {ai.path}' + ) + assert res.status == 0, f'Artifact truncation failed: {res.stderr}' + assert ( + module_capsule_configured.get_artifact_info(path=ai.path) != ai + ), 'Artifact corruption failed' + else: + raise ValueError(f'Unsupported damage type: {damage_type}') + + # Trigger desired variant of repair (verify_checksum) task. + opts = {'id': module_capsule_configured.nailgun_capsule.id} + if repair_type == 'repo': + opts.update({'repository-id': module_synced_content.repos[0].id}) + elif repair_type == 'cv': + opts.update({'content-view-id': module_synced_content.cv.id}) + elif repair_type == 'lce': + opts.update({'lifecycle-environment-id': module_synced_content.lce.id}) + module_target_sat.cli.Capsule.content_verify_checksum(opts) + + # Check if the artifact is back in shape. + fixed_ai = module_capsule_configured.get_artifact_info(path=ai.path) + assert fixed_ai == ai, f'Artifact restoration failed: {fixed_ai} != {ai}' diff --git a/tests/upgrades/test_capsule.py b/tests/upgrades/test_capsule.py index e106c9a860e..c6c76ecbde6 100644 --- a/tests/upgrades/test_capsule.py +++ b/tests/upgrades/test_capsule.py @@ -140,8 +140,8 @@ def test_post_user_scenario_capsule_sync( assert pkg in sat_files, f'{pkg=} is not in the {repo=} on satellite' assert pkg in cap_files, f'{pkg=} is not in the {repo=} on capsule' - sat_files_md5 = [target_sat.md5_by_url(url) for url in sat_files_urls] - cap_files_md5 = [target_sat.md5_by_url(url) for url in cap_files_urls] + sat_files_md5 = [target_sat.checksum_by_url(url) for url in sat_files_urls] + cap_files_md5 = [target_sat.checksum_by_url(url) for url in cap_files_urls] assert sat_files_md5 == cap_files_md5, 'satellite and capsule rpm md5sums are differrent' @@ -221,6 +221,6 @@ def test_post_user_scenario_capsule_sync_yum_repo( assert pkg in sat_files, f'{pkg=} is not in the {repo=} on satellite' assert pkg in cap_files, f'{pkg=} is not in the {repo=} on capsule' - sat_files_md5 = [target_sat.md5_by_url(url) for url in sat_files_urls] - cap_files_md5 = [target_sat.md5_by_url(url) for url in cap_files_urls] + sat_files_md5 = [target_sat.checksum_by_url(url) for url in sat_files_urls] + cap_files_md5 = [target_sat.checksum_by_url(url) for url in cap_files_urls] assert sat_files_md5 == cap_files_md5, 'satellite and capsule rpm md5sums are differrent'