feat: enhance ticket compliance analysis with human verification tracking

pr_agent/algo/utils.py

@@ -316,47 +316,98 @@ def extract_relevant_lines_str(end_line, files, relevant_file, start_line, deden
 
 def ticket_markdown_logic(emoji, markdown_text, value, gfm_supported) -> str:
     ticket_compliance_str = ""
-    final_compliance_level = -1
+    compliance_emoji = ''
+    # Track compliance levels across all tickets
+    all_compliance_levels = []
+
     if isinstance(value, list):
-        for v in value:
-            ticket_url = v.get('ticket_url', '').strip()
-            compliance_level = v.get('overall_compliance_level', '').strip()
-            # add emojis, if 'Fully compliant' ✅, 'Partially compliant' 🔶, or 'Not compliant' ❌
-            if compliance_level.lower() == 'fully compliant':
-                # compliance_level = '✅ Fully compliant'
-                final_compliance_level = 2 if final_compliance_level == -1 else 1
-            elif compliance_level.lower() == 'partially compliant':
-                # compliance_level = '🔶 Partially compliant'
-                final_compliance_level = 1
-            elif compliance_level.lower() == 'not compliant':
-                # compliance_level = '❌ Not compliant'
-                final_compliance_level = 0 if final_compliance_level < 1 else 1
-
-            # explanation = v.get('compliance_analysis', '').strip()
-            explanation = ''
-            fully_compliant_str = v.get('fully_compliant_requirements', '').strip()
-            not_compliant_str = v.get('not_compliant_requirements', '').strip()
-            if fully_compliant_str:
-                explanation += f"Fully compliant requirements:\n{fully_compliant_str}\n\n"
-            if not_compliant_str:
-                explanation += f"Not compliant requirements:\n{not_compliant_str}\n\n"
-
-            ticket_compliance_str += f"\n\n**[{ticket_url.split('/')[-1]}]({ticket_url}) - {compliance_level}**\n\n{explanation}\n\n"
-        if final_compliance_level == 2:
-            compliance_level = '✅'
-        elif final_compliance_level == 1:
-            compliance_level = '🔶'
-        else:
-            compliance_level = '❌'
+        for ticket_analysis in value:
+            try:
+                ticket_url = ticket_analysis.get('ticket_url', '').strip()
+                explanation = ''
+                ticket_compliance_level = ''  # Individual ticket compliance
+                fully_compliant_str = ticket_analysis.get('fully_compliant_requirements', '').strip()
+                not_compliant_str = ticket_analysis.get('not_compliant_requirements', '').strip()
+                requires_further_human_verification = ticket_analysis.get('requires_further_human_verification',
+                                                                          '').strip()
+
+                if not fully_compliant_str and not not_compliant_str:
+                    get_logger().debug(f"Ticket compliance has no requirements",
+                                       artifact={'ticket_url': ticket_url})
+                    continue
 
+                # Calculate individual ticket compliance level
+                if fully_compliant_str:
+                    if not_compliant_str:
+                        ticket_compliance_level = 'Partially compliant'
+                    else:
+                        if not requires_further_human_verification:
+                            ticket_compliance_level = 'Fully compliant'
+                        else:
+                            ticket_compliance_level = 'PR Code Verified'
+                elif not_compliant_str:
+                    ticket_compliance_level = 'Not compliant'
+
+                # Store the compliance level for aggregation
+                if ticket_compliance_level:
+                    all_compliance_levels.append(ticket_compliance_level)
+
+                # build compliance string
+                if fully_compliant_str:
+                    explanation += f"Compliant requirements:\n\n{fully_compliant_str}\n\n"
+                if not_compliant_str:
+                    explanation += f"Non-compliant requirements:\n\n{not_compliant_str}\n\n"
+                if requires_further_human_verification:
+                    explanation += f"Requires further human verification:\n\n{requires_further_human_verification}\n\n"
+                ticket_compliance_str += f"\n\n**[{ticket_url.split('/')[-1]}]({ticket_url}) - {ticket_compliance_level}**\n\n{explanation}\n\n"
+
+                # for debugging
+                if requires_further_human_verification:
+                    get_logger().debug(f"Ticket compliance requires further human verification",
+                                       artifact={'ticket_url': ticket_url,
+                                                 'requires_further_human_verification': requires_further_human_verification,
+                                                 'compliance_level': ticket_compliance_level})
+
+            except Exception as e:
+                get_logger().exception(f"Failed to process ticket compliance: {e}")
+                continue
+
+        # Calculate overall compliance level and emoji
+        if all_compliance_levels:
+            if all(level == 'Fully compliant' for level in all_compliance_levels):
+                compliance_level = 'Fully compliant'
+                compliance_emoji = '✅'
+            elif all(level == 'PR Code Verified' for level in all_compliance_levels):
+                compliance_level = 'PR Code Verified'
+                compliance_emoji = '✅'
+            elif any(level == 'Not compliant' for level in all_compliance_levels):
+                # If there's a mix of compliant and non-compliant tickets
+                if any(level in ['Fully compliant', 'PR Code Verified'] for level in all_compliance_levels):
+                    compliance_level = 'Partially compliant'
+                    compliance_emoji = '🔶'
+                else:
+                    compliance_level = 'Not compliant'
+                    compliance_emoji = '❌'
+            elif any(level == 'Partially compliant' for level in all_compliance_levels):
+                compliance_level = 'Partially compliant'
+                compliance_emoji = '🔶'
+            else:
+                compliance_level = 'PR Code Verified'
+                compliance_emoji = '✅'
+
+            # Set extra statistics outside the ticket loop
+            get_settings().set('config.extra_statistics', {'compliance_level': compliance_level})
+
+        # editing table row for ticket compliance analysis
         if gfm_supported:
             markdown_text += f"<tr><td>\n\n"
-            markdown_text += f"**{emoji} Ticket compliance analysis {compliance_level}**\n\n"
+            markdown_text += f"**{emoji} Ticket compliance analysis {compliance_emoji}**\n\n"
             markdown_text += ticket_compliance_str
             markdown_text += f"</td></tr>\n"
         else:
-            markdown_text += f"### {emoji} Ticket compliance analysis {compliance_level}\n\n"
-            markdown_text += ticket_compliance_str+"\n\n"
+            markdown_text += f"### {emoji} Ticket compliance analysis {compliance_emoji}\n\n"
+            markdown_text += ticket_compliance_str + "\n\n"
+
     return markdown_text
 
 

pr_agent/settings/pr_reviewer_prompts.toml

@@ -75,10 +75,10 @@ class KeyIssuesComponentLink(BaseModel):
 
 class TicketCompliance(BaseModel):
     ticket_url: str = Field(description="Ticket URL or ID")
-    ticket_requirements: str = Field(description="Repeat, in your own words, all ticket requirements, in bullet points")
-    fully_compliant_requirements: str = Field(description="A list, in bullet points, of which requirements are met by the PR code. Don't explain how the requirements are met, just list them shortly. Can be empty")
-    not_compliant_requirements: str = Field(description="A list, in bullet points, of which requirements are not met by the PR code. Don't explain how the requirements are not met, just list them shortly. Can be empty")
-    overall_compliance_level: str = Field(description="Overall give this PR one of these three values in relation to the ticket: 'Fully compliant', 'Partially compliant', or 'Not compliant'")
+    ticket_requirements: str = Field(description="Repeat, in your own words (in bullet points), all the requirements, sub-tasks, DoD, and acceptance criteria raised by the ticket")
+    fully_compliant_requirements: str = Field(description="Bullet-point list of items from the  'ticket_requirements' section above that are fulfilled by the PR code. Don't explain how the requirements are met, just list them shortly. Can be empty")
+    not_compliant_requirements: str = Field(description="Bullet-point list of items from the 'ticket_requirements' section above that are not fulfilled by the PR code. Don't explain how the requirements are not met, just list them shortly. Can be empty")
+    requires_further_human_verification: str = Field(description="Bullet-point list of items from the 'ticket_requirements' section above that cannot be assessed through code review alone, are unclear, or need further human review (e.g., browser testing, UI checks). Leave empty if all 'ticket_requirements' were marked as fully compliant or not compliant")
 {%- endif %}
 
 class Review(BaseModel):

pr_agent/tools/ticket_pr_compliance_check.py

@@ -51,6 +51,11 @@ def extract_ticket_links_from_pr_description(pr_description, repo_path, base_url
                 issue_number = match[5][1:]  # remove #
                 if issue_number.isdigit() and len(issue_number) < 5 and repo_path:
                     github_tickets.add(f'{base_url_html.strip("/")}/{repo_path}/issues/{issue_number}')
+
+        if len(github_tickets) > 3:
+            get_logger().info(f"Too many tickets found in PR description: {len(github_tickets)}")
+            # Limit the number of tickets to 3
+            github_tickets = set(list(github_tickets)[:3])
     except Exception as e:
         get_logger().error(f"Error extracting tickets error= {e}",
                            artifact={"traceback": traceback.format_exc()})