Merge pull request #859 from Codium-ai/tr/image

Tr/image

README.md

@@ -40,6 +40,12 @@ CodiumAI PR-Agent aims to help efficiently review and handle pull requests, by p
 
 ## News and Updates
 
+### April 14, 2024
+You can now ask questions about images that appear in the comment, where the entire PR is considered as the context.
+see [here](https://pr-agent-docs.codium.ai/tools/ask/#ask-on-images-using-the-pr-code-as-context) for more details.
+
+<kbd><img src="https://codium.ai/images/pr_agent/ask_images5.png" width="512"></kbd>
+
 ### March 24, 2024
 PR-Agent is now available for easy installation via [pip](https://pr-agent-docs.codium.ai/installation/locally/#using-pip-package).
 

docs/docs/tools/ask.md

@@ -7,9 +7,9 @@ It can be invoked manually by commenting on any PR:
 ```
 For example:
 
-![Ask Comment](https://codium.ai/images/pr_agent/ask_comment.png){width=768}
+![Ask Comment](https://codium.ai/images/pr_agent/ask_comment.png){width=512}
 
-![Ask](https://codium.ai/images/pr_agent/ask.png){width=768}
+![Ask](https://codium.ai/images/pr_agent/ask.png){width=512}
 
 ## Ask lines
 
@@ -18,6 +18,35 @@ You can run `/ask` on specific lines of code in the PR from the PR's diff view.
 - To select multiple lines, click on the '+' sign of the first line and then hold and drag to select the rest of the lines. 
 - write `/ask "..."` in the comment box and press `Add single comment` button.
 
-![Ask Line](https://codium.ai/images/pr_agent/Ask_line.png){width=768}
+![Ask Line](https://codium.ai/images/pr_agent/Ask_line.png){width=512}
 
 Note that the tool does not have "memory" of previous questions, and answers each question independently.
+
+## Ask on images (using the PR code as context)
+
+You can also ask questions about images that appear in the comment, where the entire PR is considered as the context. The tool will answer questions based on the images in the PR.
+The basic syntax is:
+```
+/ask "..."
+
+[Image](https://real_link_to_image)
+```
+
+Note that GitHub has a mecahnism of pasting images in comments. However, pasted image does not provide a direct link.
+To get a direct link to the image, we recommend using the following steps:
+1) send a comment that contains only the image:
+
+![Ask image1](https://codium.ai/images/pr_agent/ask_images1.png){width=512}
+
+2) quote reply to that comment:
+
+![Ask image2](https://codium.ai/images/pr_agent/ask_images2.png){width=512}
+
+3) type the question below the image:
+
+![Ask image3](https://codium.ai/images/pr_agent/ask_images3.png){width=512}
+![Ask image4](https://codium.ai/images/pr_agent/ask_images3.png){width=512}
+
+4) post the comment, and receive the answer:
+
+![Ask image5](https://codium.ai/images/pr_agent/ask_images5.png){width=512}

pr_agent/algo/ai_handlers/base_ai_handler.py

@@ -15,7 +15,7 @@ def deployment_id(self):
         pass
 
     @abstractmethod   
-    async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2):
+    async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2, img_path: str = None):
         """
         This method should be implemented to return a chat completion from the AI model.
         Args:

pr_agent/algo/ai_handlers/litellm_ai_handler.py

@@ -1,5 +1,5 @@
 import os
-
+import requests
 import boto3
 import litellm
 import openai
@@ -102,13 +102,27 @@ def deployment_id(self):
         retry=retry_if_exception_type((openai.APIError, openai.APIConnectionError, openai.Timeout)), # No retry on RateLimitError
         stop=stop_after_attempt(OPENAI_RETRIES)
     )
-    async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2):
+    async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2, img_path: str = None):
         try:
             resp, finish_reason = None, None
             deployment_id = self.deployment_id
             if self.azure:
                 model = 'azure/' + model
             messages = [{"role": "system", "content": system}, {"role": "user", "content": user}]
+            if img_path:
+                try:
+                    # check if the image link is alive
+                    r = requests.head(img_path, allow_redirects=True)
+                    if r.status_code == 404:
+                        error_msg = f"The image link is not [alive](img_path).\nPlease repost the original image as a comment, and send the question again with 'quote reply' (see [instructions](https://pr-agent-docs.codium.ai/tools/ask/#ask-on-images-using-the-pr-code-as-context))."
+                        get_logger().error(error_msg)
+                        return f"{error_msg}", "error"
+                except Exception as e:
+                    get_logger().error(f"Error fetching image: {img_path}", e)
+                    return f"Error fetching image: {img_path}", "error"
+                messages[1]["content"] = [{"type": "text", "text": messages[1]["content"]},
+                                          {"type": "image_url", "image_url": {"url": img_path}}]
+
             kwargs = {
                 "model": model,
                 "deployment_id": deployment_id,

pr_agent/servers/github_app.py

@@ -86,8 +86,13 @@ async def handle_comments_on_pr(body: Dict[str, Any],
         return {}
     comment_body = body.get("comment", {}).get("body")
     if comment_body and isinstance(comment_body, str) and not comment_body.lstrip().startswith("/"):
-        get_logger().info("Ignoring comment not starting with /")
-        return {}
+        if '/ask' in comment_body and comment_body.strip().startswith('> ![image]'):
+            comment_body_split = comment_body.split('/ask')
+            comment_body = '/ask' + comment_body_split[1] +' \n' +comment_body_split[0].strip().lstrip('>')
+            get_logger().info(f"Reformatting comment_body so command is at the beginning: {comment_body}")
+        else:
+            get_logger().info("Ignoring comment not starting with /")
+            return {}
     disable_eyes = False
     if "issue" in body and "pull_request" in body["issue"] and "url" in body["issue"]["pull_request"]:
         api_url = body["issue"]["pull_request"]["url"]

pr_agent/servers/help.py

@@ -160,16 +160,17 @@ def get_ask_usage_guide():
 /ask "..."
 ```
 
-Note that the tool does not have "memory" of previous questions, and answers each question independently.        
+Note that the tool does not have "memory" of previous questions, and answers each question independently.   
+You can ask questions about the entire PR, about specific code lines, or about an image related to the PR code changes.     
         """
-        output += "\n\n<table>"
-
-        # general
-        output += "\n\n<tr><td><details> <summary><strong> More PR-Agent commands</strong></summary><hr> \n\n"
-        output += HelpMessage.get_general_bot_help_text()
-        output += "\n\n</details></td></tr>\n\n"
-
-        output += "</table>"
+        # output += "\n\n<table>"
+        #
+        # # # general
+        # # output += "\n\n<tr><td><details> <summary><strong> More PR-Agent commands</strong></summary><hr> \n\n"
+        # # output += HelpMessage.get_general_bot_help_text()
+        # # output += "\n\n</details></td></tr>\n\n"
+        #
+        # output += "</table>"
 
         output += f"\n\nSee the [ask usage](https://pr-agent-docs.codium.ai/tools/ask/) page for a comprehensive guide on using this tool.\n\n"
 

pr_agent/settings/configuration.toml

@@ -1,7 +1,7 @@
 [config]
-model="gpt-4" # "gpt-4-0125-preview"
-model_turbo="gpt-4-0125-preview"
-fallback_models=["gpt-3.5-turbo-16k"]
+model="gpt-4-turbo-2024-04-09"
+model_turbo="gpt-4-turbo-2024-04-09"
+fallback_models=["gpt-4-0125-preview"]
 git_provider="github"
 publish_output=true
 publish_output_progress=true

pr_agent/tools/pr_questions.py

@@ -56,6 +56,12 @@ async def run(self):
         get_logger().debug("Relevant configs", artifacts=relevant_configs)
         if get_settings().config.publish_output:
             self.git_provider.publish_comment("Preparing answer...", is_temporary=True)
+
+        # identify image
+        img_path = self.idenfity_image_in_comment()
+        if img_path:
+            get_logger().debug(f"Image path identified", artifact=img_path)
+
         await retry_with_fallback_models(self._prepare_prediction)
 
         pr_comment = self._prepare_pr_answer()
@@ -71,6 +77,19 @@ async def run(self):
             self.git_provider.remove_initial_comment()
         return ""
 
+    def idenfity_image_in_comment(self):
+        img_path = ''
+        if '![image]' in self.question_str:
+            # assuming structure:
+            # /ask question ...  > ![image](img_path)
+            img_path = self.question_str.split('![image]')[1].strip().strip('()')
+            self.vars['img_path'] = img_path
+        elif 'https://' in self.question_str and ('.png' in self.question_str or 'jpg' in self.question_str): # direct image link
+            # include https:// in the image path
+            img_path = 'https://' + self.question_str.split('https://')[1]
+            self.vars['img_path'] = img_path
+        return img_path
+
     async def _prepare_prediction(self, model: str):
         self.patches_diff = get_pr_diff(self.git_provider, self.token_handler, model)
         if self.patches_diff:
@@ -86,8 +105,14 @@ async def _get_prediction(self, model: str):
         environment = Environment(undefined=StrictUndefined)
         system_prompt = environment.from_string(get_settings().pr_questions_prompt.system).render(variables)
         user_prompt = environment.from_string(get_settings().pr_questions_prompt.user).render(variables)
-        response, finish_reason = await self.ai_handler.chat_completion(model=model, temperature=0.2,
-                                                                        system=system_prompt, user=user_prompt)
+        if 'img_path' in variables:
+            img_path = self.vars['img_path']
+            response, finish_reason = await self.ai_handler.chat_completion(model=model, temperature=0.2,
+                                                                            system=system_prompt, user=user_prompt,
+                                                                            img_path=img_path)
+        else:
+            response, finish_reason = await self.ai_handler.chat_completion(model=model, temperature=0.2,
+                                                                            system=system_prompt, user=user_prompt)
         return response
 
     def _prepare_pr_answer(self) -> str: