diff --git a/README.md b/README.md index 1ac8b7da7..40453d37b 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,12 @@ CodiumAI PR-Agent aims to help efficiently review and handle pull requests, by p ## News and Updates +### April 14, 2024 +You can now ask questions about images that appear in the comment, where the entire PR is considered as the context. +see [here](https://pr-agent-docs.codium.ai/tools/ask/#ask-on-images-using-the-pr-code-as-context) for more details. + + + ### March 24, 2024 PR-Agent is now available for easy installation via [pip](https://pr-agent-docs.codium.ai/installation/locally/#using-pip-package). diff --git a/docs/docs/tools/ask.md b/docs/docs/tools/ask.md index 453807fd2..79c4f00f4 100644 --- a/docs/docs/tools/ask.md +++ b/docs/docs/tools/ask.md @@ -7,9 +7,9 @@ It can be invoked manually by commenting on any PR: ``` For example: -![Ask Comment](https://codium.ai/images/pr_agent/ask_comment.png){width=768} +![Ask Comment](https://codium.ai/images/pr_agent/ask_comment.png){width=512} -![Ask](https://codium.ai/images/pr_agent/ask.png){width=768} +![Ask](https://codium.ai/images/pr_agent/ask.png){width=512} ## Ask lines @@ -18,6 +18,35 @@ You can run `/ask` on specific lines of code in the PR from the PR's diff view. - To select multiple lines, click on the '+' sign of the first line and then hold and drag to select the rest of the lines. - write `/ask "..."` in the comment box and press `Add single comment` button. -![Ask Line](https://codium.ai/images/pr_agent/Ask_line.png){width=768} +![Ask Line](https://codium.ai/images/pr_agent/Ask_line.png){width=512} Note that the tool does not have "memory" of previous questions, and answers each question independently. + +## Ask on images (using the PR code as context) + +You can also ask questions about images that appear in the comment, where the entire PR is considered as the context. The tool will answer questions based on the images in the PR. +The basic syntax is: +``` +/ask "..." + +[Image](https://real_link_to_image) +``` + +Note that GitHub has a mecahnism of pasting images in comments. However, pasted image does not provide a direct link. +To get a direct link to the image, we recommend using the following steps: +1) send a comment that contains only the image: + +![Ask image1](https://codium.ai/images/pr_agent/ask_images1.png){width=512} + +2) quote reply to that comment: + +![Ask image2](https://codium.ai/images/pr_agent/ask_images2.png){width=512} + +3) type the question below the image: + +![Ask image3](https://codium.ai/images/pr_agent/ask_images3.png){width=512} +![Ask image4](https://codium.ai/images/pr_agent/ask_images3.png){width=512} + +4) post the comment, and receive the answer: + +![Ask image5](https://codium.ai/images/pr_agent/ask_images5.png){width=512} \ No newline at end of file diff --git a/pr_agent/algo/ai_handlers/base_ai_handler.py b/pr_agent/algo/ai_handlers/base_ai_handler.py index c8473fb3e..b5166b8e6 100644 --- a/pr_agent/algo/ai_handlers/base_ai_handler.py +++ b/pr_agent/algo/ai_handlers/base_ai_handler.py @@ -15,7 +15,7 @@ def deployment_id(self): pass @abstractmethod - async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2): + async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2, img_path: str = None): """ This method should be implemented to return a chat completion from the AI model. Args: diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index d07542f68..4acd55ea3 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -1,5 +1,5 @@ import os - +import requests import boto3 import litellm import openai @@ -102,13 +102,27 @@ def deployment_id(self): retry=retry_if_exception_type((openai.APIError, openai.APIConnectionError, openai.Timeout)), # No retry on RateLimitError stop=stop_after_attempt(OPENAI_RETRIES) ) - async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2): + async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2, img_path: str = None): try: resp, finish_reason = None, None deployment_id = self.deployment_id if self.azure: model = 'azure/' + model messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + if img_path: + try: + # check if the image link is alive + r = requests.head(img_path, allow_redirects=True) + if r.status_code == 404: + error_msg = f"The image link is not [alive](img_path).\nPlease repost the original image as a comment, and send the question again with 'quote reply' (see [instructions](https://pr-agent-docs.codium.ai/tools/ask/#ask-on-images-using-the-pr-code-as-context))." + get_logger().error(error_msg) + return f"{error_msg}", "error" + except Exception as e: + get_logger().error(f"Error fetching image: {img_path}", e) + return f"Error fetching image: {img_path}", "error" + messages[1]["content"] = [{"type": "text", "text": messages[1]["content"]}, + {"type": "image_url", "image_url": {"url": img_path}}] + kwargs = { "model": model, "deployment_id": deployment_id, diff --git a/pr_agent/servers/github_app.py b/pr_agent/servers/github_app.py index bc7042b14..f0d1340dd 100644 --- a/pr_agent/servers/github_app.py +++ b/pr_agent/servers/github_app.py @@ -86,8 +86,13 @@ async def handle_comments_on_pr(body: Dict[str, Any], return {} comment_body = body.get("comment", {}).get("body") if comment_body and isinstance(comment_body, str) and not comment_body.lstrip().startswith("/"): - get_logger().info("Ignoring comment not starting with /") - return {} + if '/ask' in comment_body and comment_body.strip().startswith('> ![image]'): + comment_body_split = comment_body.split('/ask') + comment_body = '/ask' + comment_body_split[1] +' \n' +comment_body_split[0].strip().lstrip('>') + get_logger().info(f"Reformatting comment_body so command is at the beginning: {comment_body}") + else: + get_logger().info("Ignoring comment not starting with /") + return {} disable_eyes = False if "issue" in body and "pull_request" in body["issue"] and "url" in body["issue"]["pull_request"]: api_url = body["issue"]["pull_request"]["url"] diff --git a/pr_agent/servers/help.py b/pr_agent/servers/help.py index 9e85887d1..d76f70a45 100644 --- a/pr_agent/servers/help.py +++ b/pr_agent/servers/help.py @@ -160,16 +160,17 @@ def get_ask_usage_guide(): /ask "..." ``` -Note that the tool does not have "memory" of previous questions, and answers each question independently. +Note that the tool does not have "memory" of previous questions, and answers each question independently. +You can ask questions about the entire PR, about specific code lines, or about an image related to the PR code changes. """ - output += "\n\n" - - # general - output += "\n\n\n\n" - - output += "
More PR-Agent commands
\n\n" - output += HelpMessage.get_general_bot_help_text() - output += "\n\n
" + # output += "\n\n" + # + # # # general + # # output += "\n\n\n\n" + # + # output += "
More PR-Agent commands
\n\n" + # # output += HelpMessage.get_general_bot_help_text() + # # output += "\n\n
" output += f"\n\nSee the [ask usage](https://pr-agent-docs.codium.ai/tools/ask/) page for a comprehensive guide on using this tool.\n\n" diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index 392827066..4ab511c14 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -1,7 +1,7 @@ [config] -model="gpt-4" # "gpt-4-0125-preview" -model_turbo="gpt-4-0125-preview" -fallback_models=["gpt-3.5-turbo-16k"] +model="gpt-4-turbo-2024-04-09" +model_turbo="gpt-4-turbo-2024-04-09" +fallback_models=["gpt-4-0125-preview"] git_provider="github" publish_output=true publish_output_progress=true diff --git a/pr_agent/tools/pr_questions.py b/pr_agent/tools/pr_questions.py index 4e1d3c1e6..d78d08803 100644 --- a/pr_agent/tools/pr_questions.py +++ b/pr_agent/tools/pr_questions.py @@ -56,6 +56,12 @@ async def run(self): get_logger().debug("Relevant configs", artifacts=relevant_configs) if get_settings().config.publish_output: self.git_provider.publish_comment("Preparing answer...", is_temporary=True) + + # identify image + img_path = self.idenfity_image_in_comment() + if img_path: + get_logger().debug(f"Image path identified", artifact=img_path) + await retry_with_fallback_models(self._prepare_prediction) pr_comment = self._prepare_pr_answer() @@ -71,6 +77,19 @@ async def run(self): self.git_provider.remove_initial_comment() return "" + def idenfity_image_in_comment(self): + img_path = '' + if '![image]' in self.question_str: + # assuming structure: + # /ask question ... > ![image](img_path) + img_path = self.question_str.split('![image]')[1].strip().strip('()') + self.vars['img_path'] = img_path + elif 'https://' in self.question_str and ('.png' in self.question_str or 'jpg' in self.question_str): # direct image link + # include https:// in the image path + img_path = 'https://' + self.question_str.split('https://')[1] + self.vars['img_path'] = img_path + return img_path + async def _prepare_prediction(self, model: str): self.patches_diff = get_pr_diff(self.git_provider, self.token_handler, model) if self.patches_diff: @@ -86,8 +105,14 @@ async def _get_prediction(self, model: str): environment = Environment(undefined=StrictUndefined) system_prompt = environment.from_string(get_settings().pr_questions_prompt.system).render(variables) user_prompt = environment.from_string(get_settings().pr_questions_prompt.user).render(variables) - response, finish_reason = await self.ai_handler.chat_completion(model=model, temperature=0.2, - system=system_prompt, user=user_prompt) + if 'img_path' in variables: + img_path = self.vars['img_path'] + response, finish_reason = await self.ai_handler.chat_completion(model=model, temperature=0.2, + system=system_prompt, user=user_prompt, + img_path=img_path) + else: + response, finish_reason = await self.ai_handler.chat_completion(model=model, temperature=0.2, + system=system_prompt, user=user_prompt) return response def _prepare_pr_answer(self) -> str: