From 40ee3f2be7d851698d0eed0c3359e1e9d11d1b21 Mon Sep 17 00:00:00 2001 From: Satellite QE <115476073+Satellite-QE@users.noreply.github.com> Date: Mon, 3 Jun 2024 08:42:02 -0400 Subject: [PATCH] [6.13.z] Add support for reporting/commenting test results to the Jira issue (#15274) Co-authored-by: Jameer Pathan <21165044+jameerpathan111@users.noreply.github.com> --- conf/jira.yaml.template | 5 + conftest.py | 5 +- pytest_plugins/jira_comments.py | 92 +++++++++++++++++++ robottelo/config/validators.py | 4 + robottelo/utils/issue_handlers/jira.py | 122 +++++++++++++++++-------- tests/foreman/ui/test_rhc.py | 2 +- 6 files changed, 190 insertions(+), 40 deletions(-) create mode 100644 pytest_plugins/jira_comments.py diff --git a/conf/jira.yaml.template b/conf/jira.yaml.template index e76ac35f157..9c90993fe2d 100644 --- a/conf/jira.yaml.template +++ b/conf/jira.yaml.template @@ -3,3 +3,8 @@ JIRA: URL: https://issues.redhat.com # Provide api_key to access Jira REST API API_KEY: replace-with-jira-api-key + COMMENT_TYPE: group + COMMENT_VISIBILITY: "Red Hat Employee" + ENABLE_COMMENT: false + # Comment only if jira is in one of the following state + ISSUE_STATUS: ["Review", "Release Pending"] diff --git a/conftest.py b/conftest.py index c81b56a0c93..968c5877ea5 100644 --- a/conftest.py +++ b/conftest.py @@ -22,6 +22,7 @@ 'pytest_plugins.requirements.update_requirements', 'pytest_plugins.sanity_plugin', 'pytest_plugins.video_cleanup', + 'pytest_plugins.jira_comments', 'pytest_plugins.capsule_n-minus', # Fixtures 'pytest_fixtures.core.broker', @@ -80,9 +81,9 @@ def pytest_runtest_makereport(item, call): # execute all other hooks to obtain the report object outcome = yield - rep = outcome.get_result() + report = outcome.get_result() # set a report attribute for each phase of a call, which can # be "setup", "call", "teardown" - setattr(item, "rep_" + rep.when, rep) + setattr(item, "report_" + report.when, report) diff --git a/pytest_plugins/jira_comments.py b/pytest_plugins/jira_comments.py new file mode 100644 index 00000000000..9e8d56a6843 --- /dev/null +++ b/pytest_plugins/jira_comments.py @@ -0,0 +1,92 @@ +from collections import defaultdict +import os + +import pytest + +from robottelo.config import settings +from robottelo.logging import logger +from robottelo.utils.issue_handlers.jira import add_comment_on_jira + + +def pytest_addoption(parser): + """Add --jira-comments option to report test results on the Jira issue.""" + help_comment = ( + 'Report/Comment test results on Jira issues. ' + 'Test results marked with "Verifies" or "BlockedBy" doc fields will be commented on the corresponding Jira issues. ' + 'Note: To prevent accidental use, users must set ENABLE_COMMENT to true in the jira.yaml configuration file.' + ) + parser.addoption( + '--jira-comments', + action='store_true', + default=False, + help=help_comment, + ) + + +def pytest_configure(config): + """Register jira_comments markers to avoid warnings.""" + config.addinivalue_line('markers', 'jira_comments: Add test result comment on Jira issue.') + pytest.jira_comments = config.getoption('jira_comments') + + +def update_issue_to_tests_map(item, marker, test_result): + """If the test has Verifies or BlockedBy doc field, + an issue to tests mapping will be added/updated in config.issue_to_tests_map + for each test run with outcome of the test. + """ + if marker: + for issue in marker.args[0]: + item.config.issue_to_tests_map[issue].append( + {'nodeid': item.nodeid, 'outcome': test_result} + ) + + +@pytest.hookimpl(trylast=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Create jira issue to test result mapping. Used for commenting result on Jira.""" + outcome = yield + verifies_marker = item.get_closest_marker('verifies_issues') + blocked_by_marker = item.get_closest_marker('blocked_by') + enable_jira_comments = item.config.getoption('jira_comments') + if ( + settings.jira.enable_comment + and enable_jira_comments + and (verifies_marker or blocked_by_marker) + ): + report = outcome.get_result() + if report.when == 'teardown': + test_result = ( + 'passed' + if ( + item.report_setup.passed + and item.report_call.passed + and report.outcome == 'passed' + ) + else 'failed' + ) + if not hasattr(item.config, 'issue_to_tests_map'): + item.config.issue_to_tests_map = defaultdict(list) + # Update issue_to_tests_map for Verifies testimony marker + update_issue_to_tests_map(item, verifies_marker, test_result) + # Update issue_to_tests_map for BlockedBy testimony marker + update_issue_to_tests_map(item, blocked_by_marker, test_result) + + +def pytest_sessionfinish(session, exitstatus): + """Add test result comment to related Jira issues.""" + if hasattr(session.config, 'issue_to_tests_map'): + user = os.environ.get('USER') + build_url = os.environ.get('BUILD_URL') + for issue in session.config.issue_to_tests_map: + comment_body = ( + f'This is an automated comment from job/user: {build_url if build_url else user} for a Robottelo test run.\n' + f'Satellite/Capsule: {settings.server.version.release} Snap: {settings.server.version.snap} \n' + f'Result for tests linked with issue: {issue} \n' + ) + for item in session.config.issue_to_tests_map[issue]: + comment_body += f'{item["nodeid"]} : {item["outcome"]} \n' + try: + add_comment_on_jira(issue, comment_body) + except Exception as e: + # Handle any errors in adding comments to Jira + logger.warning(f'Failed to add comment to Jira issue {issue}: {e}') diff --git a/robottelo/config/validators.py b/robottelo/config/validators.py index 1a6a6d7f4aa..494631edcb5 100644 --- a/robottelo/config/validators.py +++ b/robottelo/config/validators.py @@ -192,6 +192,10 @@ jira=[ Validator('jira.url', default='https://issues.redhat.com'), Validator('jira.api_key', must_exist=True), + Validator('jira.comment_type', default="group"), + Validator('jira.comment_visibility', default="Red Hat Employee"), + Validator('jira.enable_comment', default=False), + Validator('jira.issue_status', default=["Review", "Release Pending"]), ], ldap=[ Validator( diff --git a/robottelo/utils/issue_handlers/jira.py b/robottelo/utils/issue_handlers/jira.py index dfeb1c966c4..0a03a538952 100644 --- a/robottelo/utils/issue_handlers/jira.py +++ b/robottelo/utils/issue_handlers/jira.py @@ -21,15 +21,15 @@ VERSION_RE = re.compile(r'(?:sat-)*?(?P\d\.\d)\.\w*') -def is_open_jira(issue, data=None): +def is_open_jira(issue_id, data=None): """Check if specific Jira is open consulting a cached `data` dict or calling Jira REST API. Arguments: - issue {str} -- The Jira reference e.g: SAT-20548 + issue_id {str} -- The Jira reference e.g: SAT-20548 data {dict} -- Issue data indexed by : or None """ - jira = try_from_cache(issue, data) + jira = try_from_cache(issue_id, data) if jira.get("is_open") is not None: # issue has been already processed return jira["is_open"] @@ -54,39 +54,39 @@ def is_open_jira(issue, data=None): return status not in JIRA_CLOSED_STATUSES and status != JIRA_ONQA_STATUS -def are_all_jira_open(issues, data=None): +def are_all_jira_open(issue_ids, data=None): """Check if all Jira is open consulting a cached `data` dict or calling Jira REST API. Arguments: - issues {list} -- The Jira reference e.g: ['SAT-20548', 'SAT-20548'] + issue_ids {list} -- The Jira reference e.g: ['SAT-20548', 'SAT-20548'] data {dict} -- Issue data indexed by : or None """ - return all(is_open_jira(issue, data) for issue in issues) + return all(is_open_jira(issue_id, data) for issue_id in issue_ids) -def are_any_jira_open(issues, data=None): +def are_any_jira_open(issue_ids, data=None): """Check if any of the Jira is open consulting a cached `data` dict or calling Jira REST API. Arguments: - issues {list} -- The Jira reference e.g: ['SAT-20548', 'SAT-20548'] + issue_ids {list} -- The Jira reference e.g: ['SAT-20548', 'SAT-20548'] data {dict} -- Issue data indexed by : or None """ - return any(is_open_jira(issue, data) for issue in issues) + return any(is_open_jira(issue_id, data) for issue_id in issue_ids) -def should_deselect_jira(issue, data=None): - """Check if test should be deselected based on marked issue. +def should_deselect_jira(issue_id, data=None): + """Check if test should be deselected based on marked issue_id. 1. Resolution "Obsolete" should deselect Arguments: - issue {str} -- The Jira reference e.g: SAT-12345 + issue_id {str} -- The Jira reference e.g: SAT-12345 data {dict} -- Issue data indexed by : or None """ - jira = try_from_cache(issue, data) + jira = try_from_cache(issue_id, data) if jira.get("is_deselected") is not None: # issue has been already processed return jira["is_deselected"] @@ -105,21 +105,21 @@ def follow_duplicates(jira): return jira -def try_from_cache(issue, data=None): +def try_from_cache(issue_id, data=None): """Try to fetch issue from given data cache or previous loaded on pytest. Arguments: - issue {str} -- The Jira reference e.g: SAT-12345 + issue_id {str} -- The Jira reference e.g: SAT-12345 data {dict} -- Issue data indexed by : or None """ try: - # issue must be passed in `data` argument or already fetched in pytest - if not data and not len(pytest.issue_data[issue]['data']): + # issue_id must be passed in `data` argument or already fetched in pytest + if not data and not len(pytest.issue_data[issue_id]['data']): raise ValueError - return data or pytest.issue_data[issue]['data'] + return data or pytest.issue_data[issue_id]['data'] except (KeyError, AttributeError, ValueError): # pragma: no cover # If not then call Jira API again - return get_single_jira(str(issue)) + return get_single_jira(str(issue_id)) def collect_data_jira(collected_data, cached_data): # pragma: no cover @@ -169,26 +169,26 @@ def collect_dupes(jira, collected_data, cached_data=None): # pragma: no cover stop=stop_after_attempt(4), # Retry 3 times before raising wait=wait_fixed(20), # Wait seconds between retries ) -def get_data_jira(jira_numbers, cached_data=None): # pragma: no cover +def get_data_jira(issue_ids, cached_data=None): # pragma: no cover """Get a list of marked Jira data and query Jira REST API. Arguments: - jira_numbers {list of str} -- ['SAT-12345', ...] + issue_ids {list of str} -- ['SAT-12345', ...] cached_data {dict} -- Cached data previous loaded from API Returns: [list of dicts] -- [{'id':..., 'status':..., 'resolution': ...}] """ - if not jira_numbers: + if not issue_ids: return [] - cached_by_call = CACHED_RESPONSES['get_data'].get(str(sorted(jira_numbers))) + cached_by_call = CACHED_RESPONSES['get_data'].get(str(sorted(issue_ids))) if cached_by_call: return cached_by_call if cached_data: - logger.debug(f"Using cached data for {set(jira_numbers)}") - if not all([f'{number}' in cached_data for number in jira_numbers]): + logger.debug(f"Using cached data for {set(issue_ids)}") + if not all([f'{number}' in cached_data for number in issue_ids]): logger.debug("There are Jira's out of cache.") return [item['data'] for _, item in cached_data.items() if 'data' in item] @@ -200,10 +200,10 @@ def get_data_jira(jira_numbers, cached_data=None): # pragma: no cover "Provide api_key or a jira_cache.json." ) # Provide default data for collected Jira's. - return [get_default_jira(number) for number in jira_numbers] + return [get_default_jira(issue_id) for issue_id in issue_ids] # No cached data so Call Jira API - logger.debug(f"Calling Jira API for {set(jira_numbers)}") + logger.debug(f"Calling Jira API for {set(issue_ids)}") jira_fields = [ "key", "summary", @@ -216,7 +216,7 @@ def get_data_jira(jira_numbers, cached_data=None): # pragma: no cover assert field not in jira_fields # Generate jql - jql = ' OR '.join([f"id = {id}" for id in jira_numbers]) + jql = ' OR '.join([f"id = {issue_id}" for issue_id in issue_ids]) response = requests.get( f"{settings.jira.url}/rest/api/latest/search/", @@ -244,31 +244,79 @@ def get_data_jira(jira_numbers, cached_data=None): # pragma: no cover for issue in data if issue is not None ] - CACHED_RESPONSES['get_data'][str(sorted(jira_numbers))] = data + CACHED_RESPONSES['get_data'][str(sorted(issue_ids))] = data return data -def get_single_jira(number, cached_data=None): # pragma: no cover +def get_single_jira(issue_id, cached_data=None): # pragma: no cover """Call Jira API to get a single Jira data and cache it""" cached_data = cached_data or {} - jira_data = CACHED_RESPONSES['get_single'].get(number) + jira_data = CACHED_RESPONSES['get_single'].get(issue_id) if not jira_data: try: - jira_data = cached_data[f"{number}"]['data'] + jira_data = cached_data[f"{issue_id}"]['data'] except (KeyError, TypeError): - jira_data = get_data_jira([str(number)], cached_data) + jira_data = get_data_jira([str(issue_id)], cached_data) jira_data = jira_data and jira_data[0] - CACHED_RESPONSES['get_single'][number] = jira_data - return jira_data or get_default_jira(number) + CACHED_RESPONSES['get_single'][issue_id] = jira_data + return jira_data or get_default_jira(issue_id) -def get_default_jira(number): # pragma: no cover +def get_default_jira(issue_id): # pragma: no cover """This is the default Jira data when it is not possible to reach Jira api""" return { - "key": number, + "key": issue_id, "is_open": True, "is_deselected": False, "status": "", "resolution": "", "error": "missing jira api_key", } + + +def add_comment_on_jira( + issue_id, + comment, + comment_type=settings.jira.comment_type, + comment_visibility=settings.jira.comment_visibility, +): + """Adds a new comment to a Jira issue. + + Arguments: + issue_id {str} -- Jira issue number, ex. SAT-12232 + comment {str} -- Comment to add on the issue. + comment_type {str} -- Type of comment to add. + comment_visibility {str} -- Comment visibility. + + Returns: + [list of dicts] -- [{'id':..., 'status':..., 'resolution': ...}] + """ + # Raise a warning if any of the following option is not set. Note: It's a xor condition. + if settings.jira.enable_comment != pytest.jira_comments: + logger.warning( + 'Jira comments are currently disabled for this run. ' + 'To enable it, please set "enable_comment" to "true" in "config/jira.yaml ' + 'and provide --jira-comment pytest option."' + ) + return None + data = try_from_cache(issue_id) + if data["status"] in settings.jira.issue_status: + logger.debug(f"Adding a new comment on {issue_id} Jira issue.") + response = requests.post( + f"{settings.jira.url}/rest/api/latest/issue/{issue_id}/comment", + json={ + "body": comment, + "visibility": { + "type": comment_type, + "value": comment_visibility, + }, + }, + headers={"Authorization": f"Bearer {settings.jira.api_key}"}, + ) + response.raise_for_status() + return response.json() + logger.warning( + f"Jira comments are currently disabled for this issue because it's in {data['status']} state. " + f"Please update issue_status in jira.conf to overide this behaviour." + ) + return None diff --git a/tests/foreman/ui/test_rhc.py b/tests/foreman/ui/test_rhc.py index b9f02dcee8f..20c602dfc5f 100644 --- a/tests/foreman/ui/test_rhc.py +++ b/tests/foreman/ui/test_rhc.py @@ -78,7 +78,7 @@ def fixture_setup_rhc_satellite( rhcloud_manifest = manifester.get_manifest() module_target_sat.upload_manifest(module_rhc_org.id, rhcloud_manifest.content) yield - if request.node.rep_call.passed: + if request.node.report_call.passed: # Enable and sync required repos repo1_id = module_target_sat.api_factory.enable_sync_redhat_repo( constants.REPOS['rhel8_aps'], module_rhc_org.id