diff --git a/robottelo/host_helpers/capsule_mixins.py b/robottelo/host_helpers/capsule_mixins.py index 9e283509696..39c9ac4986f 100644 --- a/robottelo/host_helpers/capsule_mixins.py +++ b/robottelo/host_helpers/capsule_mixins.py @@ -1,6 +1,11 @@ -from datetime import datetime +from datetime import ( + datetime, + timedelta, +) import time +from dateutil.parser import parse + from robottelo.constants import PUPPET_CAPSULE_INSTALLER, PUPPET_COMMON_INSTALLER_OPTS from robottelo.logging import logger from robottelo.utils.installer import InstallerCommand @@ -66,21 +71,57 @@ def wait_for_sync(self, timeout=600, start_time=None): # Assert that a task to sync lifecycle environment to the capsule # is started (or finished already) if start_time is None: - start_time = datetime.utcnow().replace(microsecond=0) + start_time = datetime.utcnow() + # 1s margin of safety for rounding + start_time = ( + (start_time - timedelta(seconds=1)) + .replace(microsecond=0) + .strftime('%Y-%m-%d %H:%M:%S UTC') + ) logger.info(f"Waiting for capsule {self.hostname} sync to finish ...") sync_status = self.nailgun_capsule.content_get_sync() logger.info(f"Active tasks {sync_status['active_sync_tasks']}") - assert ( - len(sync_status['active_sync_tasks']) - or datetime.strptime(sync_status['last_sync_time'], '%Y-%m-%d %H:%M:%S UTC') - >= start_time - ) + ongoing_sync = True if sync_status['active_sync_tasks'] else False - # Wait till capsule sync finishes and assert the sync task succeeded + if not sync_status['last_sync_time']: # never synced + # we expect some in-progress sync task(s) ongoing + assert len(sync_status['active_sync_tasks']) > 0, ( + f'last_sync_time for capsule {self.hostname} was None (never).' + ' Expected one or more ongoing sync task(s), but found none.' + ) + else: + if parse(sync_status['last_sync_time']) < parse(start_time): + # last sync is prior to start_time, expect some ongoing sync task(s) + assert len(sync_status['active_sync_tasks']) > 0, ( + f'last_sync_time: ({sync_status["last_sync_time"]}), for capsule {self.hostname},' + f' was older than start_time: {start_time}.' + ' Expected one or more ongoing sync task(s), but found none.' + ) + # else: pass, last sync was after or same as start_time + + # Wait till capsule sync(s) finish, + # by default polled task must succeed for task in sync_status['active_sync_tasks']: self.satellite.api.ForemanTask(id=task['id']).poll(timeout=timeout) + + # fetch updated sync status, assert no failed tasks sync_status = self.nailgun_capsule.content_get_sync() - assert len(sync_status['last_failed_sync_tasks']) == 0 + assert len(sync_status['last_failed_sync_tasks']) == 0, ( + f'One or more sync tasks have failed for capsule {self.hostname},' + f' {sync_status["last_failed_sync_tasks"]}' + ) + # there were prior active sync task(s), when polled had finished: + # assert current status last_sync_time updated to on or after start_time + # assert current status shows no active sync tasks remaining + if ongoing_sync: + assert parse(sync_status['last_sync_time']) >= parse(start_time), ( + f'After polling finished in-progress sync task(s), for capsule {self.hostname},' + f' last_sync_time: {sync_status["last_sync_time"]}, was not on or after start_time: {start_time}.' + ) + assert len(sync_status['active_sync_tasks']) == 0, ( + f'Unexpected active sync task(s) remaining for capsule {self.hostname},' + f' {sync_status["active_sync_tasks"]}' + ) def get_published_repo_url(self, org, prod, repo, lce=None, cv=None): """Forms url of a repo or CV published on a Satellite or Capsule. diff --git a/tests/foreman/api/test_capsulecontent.py b/tests/foreman/api/test_capsulecontent.py index abbfd9f9f32..f43b700cb21 100644 --- a/tests/foreman/api/test_capsulecontent.py +++ b/tests/foreman/api/test_capsulecontent.py @@ -84,13 +84,13 @@ def test_positive_uploaded_content_library_sync( assert repo.read().content_counts['rpm'] == 1 # Publish new version of the content view + timestamp = datetime.utcnow() cv.publish() - cv = cv.read() + module_capsule_configured.wait_for_sync(start_time=timestamp) + cv = cv.read() assert len(cv.version) == 1 - module_capsule_configured.wait_for_sync() - # Verify the RPM published on Capsule caps_repo_url = module_capsule_configured.get_published_repo_url( org=function_org.label, @@ -149,13 +149,13 @@ def test_positive_checksum_sync( assert len(cv.version) == 1 cvv = cv.version[-1].read() + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) - cvv = cvv.read() + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - # Verify repodata's checksum type is sha256, not sha1 on capsule repo_url = module_capsule_configured.get_published_repo_url( org=function_org.label, @@ -182,13 +182,13 @@ def test_positive_checksum_sync( cv.version.sort(key=lambda version: version.id) cvv = cv.version[-1].read() + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) - cvv = cvv.read() + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - # Verify repodata's checksum type has updated to sha1 on capsule repomd = get_repomd(repo_url) checksum_types = re.findall(r'(?<=checksum type=").*?(?=")', repomd) @@ -257,12 +257,13 @@ def test_positive_sync_updated_repo( assert len(cv.version) == 1 cvv = cv.version[-1].read() + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - # Upload more content to the repository with open(DataFile.SRPM_TO_UPLOAD, 'rb') as handle: repo.upload_content(files={'content': handle}) @@ -276,12 +277,13 @@ def test_positive_sync_updated_repo( cv.version.sort(key=lambda version: version.id) cvv = cv.version[-1].read() + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - # Check the content is synced on the Capsule side properly sat_repo_url = target_sat.get_published_repo_url( org=function_org.label, @@ -358,7 +360,10 @@ def test_positive_capsule_sync( cvv = cv.version[-1].read() # Promote content view to lifecycle environment + # invoking capsule sync task(s) + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) + module_capsule_configured.wait_for_sync(start_time=timestamp) cvv = cvv.read() assert len(cvv.environment) == 2 @@ -368,8 +373,6 @@ def test_positive_capsule_sync( # repository assert repo.content_counts['rpm'] == cvv.package_count - module_capsule_configured.wait_for_sync() - # Assert that the content published on the capsule is exactly the # same as in repository on satellite sat_repo_url = target_sat.get_published_repo_url( @@ -404,14 +407,14 @@ def test_positive_capsule_sync( cv = cv.read() cv.version.sort(key=lambda version: version.id) cvv = cv.version[-1].read() - # Promote new content view version to lifecycle environment + # Promote new content view version to lifecycle environment, + # capsule sync task(s) invoked and succeed + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) - cvv = cvv.read() + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - - module_capsule_configured.wait_for_sync() - # Assert that the value of repomd revision of repository in # lifecycle environment on the capsule has not changed new_lce_revision_capsule = get_repomd_revision(caps_repo_url) @@ -427,9 +430,11 @@ def test_positive_capsule_sync( cv = cv.read() cv.version.sort(key=lambda version: version.id) cvv = cv.version[-1].read() + + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) + module_capsule_configured.wait_for_sync(start_time=timestamp) cvv = cvv.read() - assert len(cvv.environment) == 2 # Assert that packages count in the repository is updated @@ -440,8 +445,6 @@ def test_positive_capsule_sync( # repository assert repo.content_counts['rpm'] == cvv.package_count - module_capsule_configured.wait_for_sync() - # Assert that the content published on the capsule is exactly the # same as in the repository sat_files = get_repo_files_by_url(sat_repo_url) @@ -494,17 +497,17 @@ def test_positive_iso_library_sync( organization=module_entitlement_manifest_org, repository=[rh_repo] ).create() # Publish new version of the content view + timestamp = datetime.utcnow() cv.publish() - cv = cv.read() + module_capsule_configured.wait_for_sync(start_time=timestamp) + cv = cv.read() assert len(cv.version) == 1 # Verify ISOs are present on satellite sat_isos = get_repo_files_by_url(rh_repo.full_path, extension='iso') assert len(sat_isos) == 4 - module_capsule_configured.wait_for_sync() - # Verify all the ISOs are present on capsule caps_path = ( f'{module_capsule_configured.url}/pulp/content/{module_entitlement_manifest_org.label}' @@ -573,13 +576,13 @@ def test_positive_on_demand_sync( cvv = cv.version[-1].read() # Promote content view to lifecycle environment + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) - cvv = cvv.read() + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - # Verify packages on Capsule match the source caps_repo_url = module_capsule_configured.get_published_repo_url( org=function_org.label, @@ -655,13 +658,13 @@ def test_positive_update_with_immediate_sync( cvv = cv.version[-1].read() # Promote content view to lifecycle environment + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) - cvv = cvv.read() + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - # Update download policy to 'immediate' repo.download_policy = 'immediate' repo = repo.update(['download_policy']) @@ -683,13 +686,13 @@ def test_positive_update_with_immediate_sync( cv.version.sort(key=lambda version: version.id) cvv = cv.version[-1].read() # Promote content view to lifecycle environment + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) - cvv = cvv.read() + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - # Verify the count of RPMs published on Capsule caps_repo_url = module_capsule_configured.get_published_repo_url( org=function_org.label, @@ -788,13 +791,13 @@ def test_positive_sync_kickstart_repo( cvv = cv.version[-1].read() # Promote content view to lifecycle environment + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': lce.id}) - cvv = cvv.read() + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - # Check for kickstart content on SAT and CAPS tail = ( f'rhel/server/7/{constants.REPOS["kickstart"][distro]["version"]}/x86_64/kickstart' @@ -887,12 +890,13 @@ def test_positive_sync_container_repo_end_to_end( # Promote the latest CV version into capsule's LCE cvv = cv.version[-1].read() + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - # Pull the images from capsule to the content host repo_paths = [ ( @@ -1005,10 +1009,11 @@ def test_positive_sync_collection_repo( assert function_lce_library.id in [capsule_lce['id'] for capsule_lce in result['results']] # Sync the repo + timestamp = datetime.utcnow() repo.sync(timeout=600) repo = repo.read() assert repo.content_counts['ansible_collection'] == 2 - module_capsule_configured.wait_for_sync() + module_capsule_configured.wait_for_sync(start_time=timestamp) repo_path = repo.full_path.replace(target_sat.hostname, module_capsule_configured.hostname) coll_path = './collections' @@ -1087,12 +1092,13 @@ def test_positive_sync_file_repo( # Promote the latest CV version into capsule's LCE cvv = cv.version[-1].read() + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - # Run one more sync, check for status (BZ#1985122) sync_status = module_capsule_configured.nailgun_capsule.content_sync() assert sync_status['result'] == 'success' @@ -1183,16 +1189,20 @@ def test_positive_sync_CV_to_multiple_LCEs( # Promote the CV to both Capsule's LCEs without waiting for Capsule sync task completion. cvv = cv.version[-1].read() + assert len(cvv.environment) == 1 + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': lce1.id}) + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': lce2.id}) + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 3 - # Check all sync tasks finished without errors. - module_capsule_configured.wait_for_sync() - @pytest.mark.tier4 @pytest.mark.skip_if_not_set('capsule') def test_positive_capsule_sync_status_persists( @@ -1235,7 +1245,7 @@ def test_positive_capsule_sync_status_persists( cvv = cv.version[-1].read() timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) - module_capsule_configured.wait_for_sync() + module_capsule_configured.wait_for_sync(start_time=timestamp) # Delete all capsule sync tasks so that we fall back for audits. task_result = target_sat.execute( @@ -1411,12 +1421,13 @@ def test_positive_capsule_sync_openstack_container_repos( # Promote the latest CV version into capsule's LCE cvv = cv.version[-1].read() + timestamp = datetime.utcnow() cvv.promote(data={'environment_ids': function_lce.id}) + module_capsule_configured.wait_for_sync(start_time=timestamp) + cvv = cvv.read() assert len(cvv.environment) == 2 - module_capsule_configured.wait_for_sync() - @pytest.mark.parametrize( 'repos_collection', [