diff --git a/pr_agent/algo/utils.py b/pr_agent/algo/utils.py index f7a7d686c..d4fc78cf8 100644 --- a/pr_agent/algo/utils.py +++ b/pr_agent/algo/utils.py @@ -316,47 +316,98 @@ def extract_relevant_lines_str(end_line, files, relevant_file, start_line, deden def ticket_markdown_logic(emoji, markdown_text, value, gfm_supported) -> str: ticket_compliance_str = "" - final_compliance_level = -1 + compliance_emoji = '' + # Track compliance levels across all tickets + all_compliance_levels = [] + if isinstance(value, list): - for v in value: - ticket_url = v.get('ticket_url', '').strip() - compliance_level = v.get('overall_compliance_level', '').strip() - # add emojis, if 'Fully compliant' ✅, 'Partially compliant' 🔶, or 'Not compliant' ❌ - if compliance_level.lower() == 'fully compliant': - # compliance_level = '✅ Fully compliant' - final_compliance_level = 2 if final_compliance_level == -1 else 1 - elif compliance_level.lower() == 'partially compliant': - # compliance_level = '🔶 Partially compliant' - final_compliance_level = 1 - elif compliance_level.lower() == 'not compliant': - # compliance_level = '❌ Not compliant' - final_compliance_level = 0 if final_compliance_level < 1 else 1 - - # explanation = v.get('compliance_analysis', '').strip() - explanation = '' - fully_compliant_str = v.get('fully_compliant_requirements', '').strip() - not_compliant_str = v.get('not_compliant_requirements', '').strip() - if fully_compliant_str: - explanation += f"Fully compliant requirements:\n{fully_compliant_str}\n\n" - if not_compliant_str: - explanation += f"Not compliant requirements:\n{not_compliant_str}\n\n" - - ticket_compliance_str += f"\n\n**[{ticket_url.split('/')[-1]}]({ticket_url}) - {compliance_level}**\n\n{explanation}\n\n" - if final_compliance_level == 2: - compliance_level = '✅' - elif final_compliance_level == 1: - compliance_level = '🔶' - else: - compliance_level = '❌' + for ticket_analysis in value: + try: + ticket_url = ticket_analysis.get('ticket_url', '').strip() + explanation = '' + ticket_compliance_level = '' # Individual ticket compliance + fully_compliant_str = ticket_analysis.get('fully_compliant_requirements', '').strip() + not_compliant_str = ticket_analysis.get('not_compliant_requirements', '').strip() + requires_further_human_verification = ticket_analysis.get('requires_further_human_verification', + '').strip() + + if not fully_compliant_str and not not_compliant_str: + get_logger().debug(f"Ticket compliance has no requirements", + artifact={'ticket_url': ticket_url}) + continue + # Calculate individual ticket compliance level + if fully_compliant_str: + if not_compliant_str: + ticket_compliance_level = 'Partially compliant' + else: + if not requires_further_human_verification: + ticket_compliance_level = 'Fully compliant' + else: + ticket_compliance_level = 'PR Code Verified' + elif not_compliant_str: + ticket_compliance_level = 'Not compliant' + + # Store the compliance level for aggregation + if ticket_compliance_level: + all_compliance_levels.append(ticket_compliance_level) + + # build compliance string + if fully_compliant_str: + explanation += f"Compliant requirements:\n\n{fully_compliant_str}\n\n" + if not_compliant_str: + explanation += f"Non-compliant requirements:\n\n{not_compliant_str}\n\n" + if requires_further_human_verification: + explanation += f"Requires further human verification:\n\n{requires_further_human_verification}\n\n" + ticket_compliance_str += f"\n\n**[{ticket_url.split('/')[-1]}]({ticket_url}) - {ticket_compliance_level}**\n\n{explanation}\n\n" + + # for debugging + if requires_further_human_verification: + get_logger().debug(f"Ticket compliance requires further human verification", + artifact={'ticket_url': ticket_url, + 'requires_further_human_verification': requires_further_human_verification, + 'compliance_level': ticket_compliance_level}) + + except Exception as e: + get_logger().exception(f"Failed to process ticket compliance: {e}") + continue + + # Calculate overall compliance level and emoji + if all_compliance_levels: + if all(level == 'Fully compliant' for level in all_compliance_levels): + compliance_level = 'Fully compliant' + compliance_emoji = '✅' + elif all(level == 'PR Code Verified' for level in all_compliance_levels): + compliance_level = 'PR Code Verified' + compliance_emoji = '✅' + elif any(level == 'Not compliant' for level in all_compliance_levels): + # If there's a mix of compliant and non-compliant tickets + if any(level in ['Fully compliant', 'PR Code Verified'] for level in all_compliance_levels): + compliance_level = 'Partially compliant' + compliance_emoji = '🔶' + else: + compliance_level = 'Not compliant' + compliance_emoji = '❌' + elif any(level == 'Partially compliant' for level in all_compliance_levels): + compliance_level = 'Partially compliant' + compliance_emoji = '🔶' + else: + compliance_level = 'PR Code Verified' + compliance_emoji = '✅' + + # Set extra statistics outside the ticket loop + get_settings().set('config.extra_statistics', {'compliance_level': compliance_level}) + + # editing table row for ticket compliance analysis if gfm_supported: markdown_text += f"\n\n" - markdown_text += f"**{emoji} Ticket compliance analysis {compliance_level}**\n\n" + markdown_text += f"**{emoji} Ticket compliance analysis {compliance_emoji}**\n\n" markdown_text += ticket_compliance_str markdown_text += f"\n" else: - markdown_text += f"### {emoji} Ticket compliance analysis {compliance_level}\n\n" - markdown_text += ticket_compliance_str+"\n\n" + markdown_text += f"### {emoji} Ticket compliance analysis {compliance_emoji}\n\n" + markdown_text += ticket_compliance_str + "\n\n" + return markdown_text diff --git a/pr_agent/settings/pr_reviewer_prompts.toml b/pr_agent/settings/pr_reviewer_prompts.toml index 816416ec1..6e0c135c2 100644 --- a/pr_agent/settings/pr_reviewer_prompts.toml +++ b/pr_agent/settings/pr_reviewer_prompts.toml @@ -75,10 +75,10 @@ class KeyIssuesComponentLink(BaseModel): class TicketCompliance(BaseModel): ticket_url: str = Field(description="Ticket URL or ID") - ticket_requirements: str = Field(description="Repeat, in your own words, all ticket requirements, in bullet points") - fully_compliant_requirements: str = Field(description="A list, in bullet points, of which requirements are met by the PR code. Don't explain how the requirements are met, just list them shortly. Can be empty") - not_compliant_requirements: str = Field(description="A list, in bullet points, of which requirements are not met by the PR code. Don't explain how the requirements are not met, just list them shortly. Can be empty") - overall_compliance_level: str = Field(description="Overall give this PR one of these three values in relation to the ticket: 'Fully compliant', 'Partially compliant', or 'Not compliant'") + ticket_requirements: str = Field(description="Repeat, in your own words (in bullet points), all the requirements, sub-tasks, DoD, and acceptance criteria raised by the ticket") + fully_compliant_requirements: str = Field(description="Bullet-point list of items from the 'ticket_requirements' section above that are fulfilled by the PR code. Don't explain how the requirements are met, just list them shortly. Can be empty") + not_compliant_requirements: str = Field(description="Bullet-point list of items from the 'ticket_requirements' section above that are not fulfilled by the PR code. Don't explain how the requirements are not met, just list them shortly. Can be empty") + requires_further_human_verification: str = Field(description="Bullet-point list of items from the 'ticket_requirements' section above that cannot be assessed through code review alone, are unclear, or need further human review (e.g., browser testing, UI checks). Leave empty if all 'ticket_requirements' were marked as fully compliant or not compliant") {%- endif %} class Review(BaseModel): diff --git a/pr_agent/tools/ticket_pr_compliance_check.py b/pr_agent/tools/ticket_pr_compliance_check.py index 05cd64fe5..54c72eb97 100644 --- a/pr_agent/tools/ticket_pr_compliance_check.py +++ b/pr_agent/tools/ticket_pr_compliance_check.py @@ -51,6 +51,11 @@ def extract_ticket_links_from_pr_description(pr_description, repo_path, base_url issue_number = match[5][1:] # remove # if issue_number.isdigit() and len(issue_number) < 5 and repo_path: github_tickets.add(f'{base_url_html.strip("/")}/{repo_path}/issues/{issue_number}') + + if len(github_tickets) > 3: + get_logger().info(f"Too many tickets found in PR description: {len(github_tickets)}") + # Limit the number of tickets to 3 + github_tickets = set(list(github_tickets)[:3]) except Exception as e: get_logger().error(f"Error extracting tickets error= {e}", artifact={"traceback": traceback.format_exc()})