From 917028ad7f476d7f3044bd72e65f8caa5d883bfb Mon Sep 17 00:00:00 2001 From: Vladimir Sedmik Date: Thu, 25 Apr 2024 16:18:36 +0200 Subject: [PATCH] Add test case to verify artifacts repair at Capsule --- robottelo/cli/capsule.py | 8 ++ robottelo/constants/__init__.py | 2 +- robottelo/host_helpers/capsule_mixins.py | 25 +++- robottelo/host_helpers/satellite_mixins.py | 12 +- tests/foreman/api/test_capsulecontent.py | 23 ++-- tests/foreman/api/test_repository.py | 4 +- tests/foreman/cli/test_capsulecontent.py | 153 ++++++++++++++++++++- tests/upgrades/test_capsule.py | 8 +- 8 files changed, 208 insertions(+), 27 deletions(-) diff --git a/robottelo/cli/capsule.py b/robottelo/cli/capsule.py index bb6825cc783..5d98eb71578 100644 --- a/robottelo/cli/capsule.py +++ b/robottelo/cli/capsule.py @@ -99,6 +99,14 @@ def content_update_counts(cls, options): return cls.execute(cls._construct_command(options), output_format='json') + @classmethod + def content_verify_checksum(cls, options): + """Trigger verify checksum task.""" + + cls.command_sub = 'content verify-checksum' + + return cls.execute(cls._construct_command(options), output_format='json') + @classmethod def import_classes(cls, options): """Import puppet classes from puppet Capsule.""" diff --git a/robottelo/constants/__init__.py b/robottelo/constants/__init__.py index 78d32535efc..ba9e16564da 100644 --- a/robottelo/constants/__init__.py +++ b/robottelo/constants/__init__.py @@ -724,7 +724,6 @@ class Colored(Box): CUSTOM_LOCAL_FILE = '/var/lib/pulp/imports/myrepo/test.txt' CUSTOM_FILE_REPO_FILES_COUNT = 3 CUSTOM_RPM_SHA_512_FEED_COUNT = {'rpm': 35, 'errata': 4} -CUSTOM_REPODATA_PATH = '/var/lib/pulp/published/yum/https/repos' CERT_PATH = "/etc/pki/ca-trust/source/anchors/" CERT_DATA = { 'capsule_hostname': 'capsule.example.com', @@ -869,6 +868,7 @@ class Colored(Box): } CUSTOM_PUPPET_MODULE_REPOS_VERSION = '-0.2.0.tar.gz' +PULP_ARTIFACT_DIR = '/var/lib/pulp/media/artifact/' PULP_EXPORT_DIR = '/var/lib/pulp/exports/' PULP_IMPORT_DIR = '/var/lib/pulp/imports/' EXPORT_LIBRARY_NAME = 'Export-Library' diff --git a/robottelo/host_helpers/capsule_mixins.py b/robottelo/host_helpers/capsule_mixins.py index 0e7ffd21065..981e8a5dd6d 100644 --- a/robottelo/host_helpers/capsule_mixins.py +++ b/robottelo/host_helpers/capsule_mixins.py @@ -1,9 +1,14 @@ from datetime import datetime, timedelta import time +from box import Box from dateutil.parser import parse -from robottelo.constants import PUPPET_CAPSULE_INSTALLER, PUPPET_COMMON_INSTALLER_OPTS +from robottelo.constants import ( + PULP_ARTIFACT_DIR, + PUPPET_CAPSULE_INSTALLER, + PUPPET_COMMON_INSTALLER_OPTS, +) from robottelo.logging import logger from robottelo.utils.installer import InstallerCommand @@ -149,3 +154,21 @@ def get_published_repo_url(self, org, prod, repo, lce=None, cv=None): if lce and cv: return f'{self.url}/pulp/content/{org}/{lce}/{cv}/custom/{prod}/{repo}/' return f'{self.url}/pulp/content/{org}/Library/custom/{prod}/{repo}/' + + def get_artifact_info(self, checksum): + """Returns information about pulp artifact if found on FS, + throws FileNotFoundError otherwise. + + :param checksum: Checksum of the artifact to look for. + :return: A Box with artifact path and info. + """ + path = f'{PULP_ARTIFACT_DIR}{checksum[0:2]}/{checksum[2:]}' + + res = self.execute(f'stat --format %s {path}') + if res.status: + raise FileNotFoundError(f'Artifact not found: {path}') + size = int(res.stdout) + real_sum = self.execute(f'sha256sum {path}').stdout.split()[0] + info = self.execute(f'file {path}').stdout.strip().split(': ')[1] + + return Box(path=path, size=size, sum=real_sum, info=info) diff --git a/robottelo/host_helpers/satellite_mixins.py b/robottelo/host_helpers/satellite_mixins.py index d20a70d2d15..0b564f04efa 100644 --- a/robottelo/host_helpers/satellite_mixins.py +++ b/robottelo/host_helpers/satellite_mixins.py @@ -116,13 +116,15 @@ def get_repomd_revision(self, repo_url): return match.group(0) - def md5_by_url(self, url): - """Returns md5 checksum of a file, accessible via URL. Useful when you want + def checksum_by_url(self, url, sum_type='md5sum'): + """Returns desired checksum of a file, accessible via URL. Useful when you want to calculate checksum but don't want to deal with storing a file and removing it afterwards. :param str url: URL of a file. - :return str: string containing md5 checksum. + :param str sum_type: Checksum type like md5sum, sha256sum, sha512sum, etc. + Defaults to md5sum. + :return str: string containing the checksum. :raises: AssertionError: If non-zero return code received (file couldn't be reached or calculation was not successful). """ @@ -131,8 +133,8 @@ def md5_by_url(self, url): if result.status != 0: raise AssertionError(f'Failed to get `{filename}` from `{url}`.') return self.execute( - f'wget -qO - {url} | tee {filename} | md5sum | awk \'{{print $1}}\'' - ).stdout + f'wget -qO - {url} | tee {filename} | {sum_type} | awk \'{{print $1}}\'' + ).stdout.strip() def upload_manifest(self, org_id, manifest=None, interface='API', timeout=None): """Upload a manifest using the requested interface. diff --git a/tests/foreman/api/test_capsulecontent.py b/tests/foreman/api/test_capsulecontent.py index 818a808a4c2..b6a411ed45b 100644 --- a/tests/foreman/api/test_capsulecontent.py +++ b/tests/foreman/api/test_capsulecontent.py @@ -37,6 +37,7 @@ FAKE_FILE_NEW_NAME, KICKSTART_CONTENT, PRDS, + PULP_ARTIFACT_DIR, REPOS, REPOSET, RH_CONTAINER_REGISTRY_HUB, @@ -635,9 +636,9 @@ def test_positive_on_demand_sync( assert len(caps_files) == packages_count # Download a package from the Capsule and get its md5 checksum - published_package_md5 = target_sat.md5_by_url(f'{caps_repo_url}/{package}') + published_package_md5 = target_sat.checksum_by_url(f'{caps_repo_url}/{package}') # Get md5 checksum of source package - package_md5 = target_sat.md5_by_url(f'{repo_url}/{package}') + package_md5 = target_sat.checksum_by_url(f'{repo_url}/{package}') # Assert checksums are matching assert package_md5 == published_package_md5 @@ -847,8 +848,10 @@ def test_positive_sync_kickstart_repo( # Check kickstart specific files for file in KICKSTART_CONTENT: - sat_file = target_sat.md5_by_url(f'{target_sat.url}/{url_base}/{file}') - caps_file = target_sat.md5_by_url(f'{module_capsule_configured.url}/{url_base}/{file}') + sat_file = target_sat.checksum_by_url(f'{target_sat.url}/{url_base}/{file}') + caps_file = target_sat.checksum_by_url( + f'{module_capsule_configured.url}/{url_base}/{file}' + ) assert sat_file == caps_file # Check packages @@ -1162,8 +1165,8 @@ def test_positive_sync_file_repo( assert sat_files == caps_files for file in sat_files: - sat_file = target_sat.md5_by_url(f'{sat_repo_url}{file}') - caps_file = target_sat.md5_by_url(f'{caps_repo_url}{file}') + sat_file = target_sat.checksum_by_url(f'{sat_repo_url}{file}') + caps_file = target_sat.checksum_by_url(f'{caps_repo_url}{file}') assert sat_file == caps_file @pytest.mark.tier4 @@ -1370,9 +1373,7 @@ def test_positive_remove_capsule_orphans( assert sync_status['result'] == 'success', 'Capsule sync task failed.' # Ensure the RPM artifacts were created. - result = capsule_configured.execute( - 'ls /var/lib/pulp/media/artifact/*/* | xargs file | grep RPM' - ) + result = capsule_configured.execute(f'ls {PULP_ARTIFACT_DIR}*/* | xargs file | grep RPM') assert not result.status, 'RPM artifacts are missing after capsule sync.' # Remove the Library LCE from the capsule and resync it. @@ -1401,9 +1402,7 @@ def test_positive_remove_capsule_orphans( ) # Ensure the artifacts were removed. - result = capsule_configured.execute( - 'ls /var/lib/pulp/media/artifact/*/* | xargs file | grep RPM' - ) + result = capsule_configured.execute(f'ls {PULP_ARTIFACT_DIR}*/* | xargs file | grep RPM') assert result.status, 'RPM artifacts are still present. They should be gone.' @pytest.mark.skip_if_not_set('capsule') diff --git a/tests/foreman/api/test_repository.py b/tests/foreman/api/test_repository.py index e253b9c957f..db6ac74ce83 100644 --- a/tests/foreman/api/test_repository.py +++ b/tests/foreman/api/test_repository.py @@ -1249,12 +1249,12 @@ def test_positive_sync_with_treeinfo_ignore( repo = repo.update(['mirroring_policy', 'ignorable_content']) repo.sync() with pytest.raises(AssertionError): - target_sat.md5_by_url(f'{repo.full_path}.treeinfo') + target_sat.checksum_by_url(f'{repo.full_path}.treeinfo') repo.ignorable_content = [] repo = repo.update(['ignorable_content']) repo.sync() - assert target_sat.md5_by_url( + assert target_sat.checksum_by_url( f'{repo.full_path}.treeinfo' ), 'The treeinfo file is missing in the KS repo but it should be there.' diff --git a/tests/foreman/cli/test_capsulecontent.py b/tests/foreman/cli/test_capsulecontent.py index 73e42659fa2..47146959a35 100644 --- a/tests/foreman/cli/test_capsulecontent.py +++ b/tests/foreman/cli/test_capsulecontent.py @@ -11,6 +11,10 @@ :CaseImportance: High """ +from datetime import datetime +import random + +from box import Box import pytest from robottelo.config import settings @@ -19,6 +23,51 @@ CONTAINER_UPSTREAM_NAME, ) from robottelo.constants.repos import ANSIBLE_GALAXY, CUSTOM_FILE_REPO +from robottelo.content_info import get_repo_files_urls_by_url + + +@pytest.fixture(scope='module') +def module_synced_content( + request, + module_target_sat, + module_capsule_configured, + module_org, + module_product, + module_lce, + module_lce_library, +): + """ + Create and sync one or more repositories, publish them in a CV, + promote to an LCE and sync all of that to an external Capsule. + + :param request: Repo(s) to use - dict or list of dicts with options to create the repo(s). + :return: Box with created instances and Capsule sync time. + """ + if not isinstance(request.param, list): + request.param = [request.param] + + repos = [] + for item in request.param: + repo = module_target_sat.api.Repository(product=module_product, **item).create() + repo.sync() + repos.append(repo) + + cv = module_target_sat.api.ContentView(organization=module_org, repository=repos).create() + cv.publish() + cvv = cv.read().version[0].read() + cvv.promote(data={'environment_ids': module_lce.id}) + + # Assign the Capsule with the LCE and sync it. + module_capsule_configured.nailgun_capsule.content_add_lifecycle_environment( + data={'environment_id': [module_lce.id, module_lce_library.id]} + ) + sync_time = datetime.utcnow() + module_target_sat.cli.Capsule.content_synchronize( + {'id': module_capsule_configured.nailgun_capsule.id, 'organization-id': module_org.id} + ) + module_capsule_configured.wait_for_sync(start_time=sync_time) + + return Box(prod=module_product, repos=repos, cv=cv, lce=module_lce, sync_time=sync_time) @pytest.mark.parametrize( @@ -43,7 +92,6 @@ ], indirect=True, ) -@pytest.mark.stream def test_positive_content_counts_for_mixed_cv( target_sat, module_capsule_configured, @@ -179,7 +227,6 @@ def test_positive_content_counts_for_mixed_cv( assert len(info['lifecycle-environments']) == 0, 'The LCE is still listed' -@pytest.mark.stream def test_positive_update_counts(target_sat, module_capsule_configured): """Verify the update counts functionality @@ -204,3 +251,105 @@ def test_positive_update_counts(target_sat, module_capsule_configured): search_rate=5, max_tries=5, ) + + +@pytest.mark.stream +@pytest.mark.parametrize('repair_type', ['repo', 'cv', 'lce']) +@pytest.mark.parametrize( + 'module_synced_content', + [ + [ + {'content_type': 'yum', 'url': settings.repos.yum_0.url}, + {'content_type': 'file', 'url': CUSTOM_FILE_REPO}, + ] + ], + indirect=True, + ids=['content'], +) +@pytest.mark.parametrize('content_type', ['yum', 'file']) +@pytest.mark.parametrize('damage_type', ['destroy', 'corrupt']) +def test_positive_repair_yum_file_artifacts( + module_target_sat, + module_capsule_configured, + module_org, + module_synced_content, + damage_type, + repair_type, + content_type, +): + """Test the verify-checksum task repairs particular RPM and FILE artifacts correctly + at the Capsule side using one of its methods when they were removed or corrupted before. + + :id: f818f537-94b0-4d14-adf1-643ead828ade + + :parametrized: yes + + :setup: + 1. Have a Satellite with registered external Capsule. + 2. Create yum and file type repository, publish it in a CVV and promote to an LCE. + 3. Assign the Capsule with the LCE and sync it. + + :steps: + 1. Pick one of the published files. + 2. Cause desired type of damage to his artifact and verify the effect. + 3. Trigger desired variant of repair (verify_checksum) task. + 4. Check if the artifact is back in shape. + + :expectedresults: + 1. Artifact is stored correctly based on the checksum. + 2. All variants of verify_checksum task are able to repair all types of damage. + + :BZ: 2127537 + + :customerscenario: true + + """ + repo = next(repo for repo in module_synced_content.repos if repo.content_type == content_type) + + # Pick one of the published files. + caps_repo_url = module_capsule_configured.get_published_repo_url( + org=module_org.label, + lce=None if repair_type == 'repo' else module_synced_content.lce.label, + cv=None if repair_type == 'repo' else module_synced_content.cv.label, + prod=module_synced_content.prod.label, + repo=repo.label, + ) + cap_files_urls = get_repo_files_urls_by_url( + caps_repo_url, extension='rpm' if content_type == 'yum' else 'iso' + ) + + file_url = random.choice(cap_files_urls) + file_sum = module_target_sat.checksum_by_url(file_url, sum_type='sha256sum') + file_ai = module_capsule_configured.get_artifact_info(file_sum) + + # Cause desired type of damage to his artifact and verify the effect. + if damage_type == 'destroy': + module_capsule_configured.execute(f'rm -f {file_ai.path}') + with pytest.raises(FileNotFoundError): + module_capsule_configured.get_artifact_info(file_sum) + with pytest.raises(AssertionError): + module_target_sat.checksum_by_url(file_url) + elif damage_type == 'corrupt': + res = module_capsule_configured.execute( + f'truncate -s {random.randint(1, file_ai.size)} {file_ai.path}' + ) + assert res.status == 0, f'Artifact truncation failed: {res.stderr}' + assert ( + module_capsule_configured.get_artifact_info(file_sum) != file_ai + ), 'Artifact corruption failed' + else: + raise ValueError(f'Unsupported damage type: {damage_type}') + + # Trigger desired variant of repair (verify_checksum) task. + opts = {'id': module_capsule_configured.nailgun_capsule.id} + if repair_type == 'repo': + opts.update({'repository-id': repo.id}) + elif repair_type == 'cv': + opts.update({'content-view-id': module_synced_content.cv.id}) + elif repair_type == 'lce': + opts.update({'lifecycle-environment-id': module_synced_content.lce.id}) + module_target_sat.cli.Capsule.content_verify_checksum(opts) + + # Check if the artifact is back in shape. + fixed_ai = module_capsule_configured.get_artifact_info(file_sum) + assert fixed_ai == file_ai, f'Artifact restoration failed: {fixed_ai} != {file_ai}' diff --git a/tests/upgrades/test_capsule.py b/tests/upgrades/test_capsule.py index c7cf63e9977..4f81c9594d3 100644 --- a/tests/upgrades/test_capsule.py +++ b/tests/upgrades/test_capsule.py @@ -139,8 +139,8 @@ def test_post_user_scenario_capsule_sync( assert pkg in sat_files, f'{pkg=} is not in the {repo=} on satellite' assert pkg in cap_files, f'{pkg=} is not in the {repo=} on capsule' - sat_files_md5 = [target_sat.md5_by_url(url) for url in sat_files_urls] - cap_files_md5 = [target_sat.md5_by_url(url) for url in cap_files_urls] + sat_files_md5 = [target_sat.checksum_by_url(url) for url in sat_files_urls] + cap_files_md5 = [target_sat.checksum_by_url(url) for url in cap_files_urls] assert sat_files_md5 == cap_files_md5, 'satellite and capsule rpm md5sums are differrent' @@ -220,6 +220,6 @@ def test_post_user_scenario_capsule_sync_yum_repo( assert pkg in sat_files, f'{pkg=} is not in the {repo=} on satellite' assert pkg in cap_files, f'{pkg=} is not in the {repo=} on capsule' - sat_files_md5 = [target_sat.md5_by_url(url) for url in sat_files_urls] - cap_files_md5 = [target_sat.md5_by_url(url) for url in cap_files_urls] + sat_files_md5 = [target_sat.checksum_by_url(url) for url in sat_files_urls] + cap_files_md5 = [target_sat.checksum_by_url(url) for url in cap_files_urls] assert sat_files_md5 == cap_files_md5, 'satellite and capsule rpm md5sums are differrent'