Add guard prompt to the chain

app/llm/external/model.py

@@ -1,4 +1,6 @@
 from abc import ABCMeta, abstractmethod
+from typing import Optional
+
 from pydantic import BaseModel
 
 from ...domain import IrisMessage
@@ -12,7 +14,7 @@ class LanguageModel(BaseModel, metaclass=ABCMeta):
     id: str
     name: str
     description: str
-    capabilities: CapabilityList
+    capabilities: Optional[CapabilityList] = None
 
 
 class CompletionModel(LanguageModel, metaclass=ABCMeta):

app/llm/langchain/iris_langchain_chat_model.py

@@ -19,9 +19,17 @@ class IrisLangchainChatModel(BaseChatModel):
     """Custom langchain chat model for our own request handler"""
 
     request_handler: RequestHandler
+    completion_args: CompletionArguments
 
-    def __init__(self, request_handler: RequestHandler, **kwargs: Any) -> None:
-        super().__init__(request_handler=request_handler, **kwargs)
+    def __init__(
+        self,
+        request_handler: RequestHandler,
+        completion_args: Optional[CompletionArguments] = CompletionArguments(stop=None),
+        **kwargs: Any
+    ) -> None:
+        super().__init__(
+            request_handler=request_handler, completion_args=completion_args, **kwargs
+        )
 
     def _generate(
         self,
@@ -31,9 +39,8 @@ def _generate(
         **kwargs: Any
     ) -> ChatResult:
         iris_messages = [convert_langchain_message_to_iris_message(m) for m in messages]
-        iris_message = self.request_handler.chat(
-            iris_messages, CompletionArguments(stop=stop)
-        )
+        self.completion_args.stop = stop
+        iris_message = self.request_handler.chat(iris_messages, self.completion_args)
         base_message = convert_iris_message_to_langchain_message(iris_message)
         chat_generation = ChatGeneration(message=base_message)
         return ChatResult(generations=[chat_generation])

app/pipeline/chat/file_selector_pipeline.py

@@ -3,12 +3,11 @@
 from typing import Dict, Optional, List
 
 from langchain.output_parsers import PydanticOutputParser
-from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import PromptTemplate, ChatPromptTemplate
 from langchain_core.runnables import Runnable
 from pydantic import BaseModel
 
-from ...llm import BasicRequestHandler
+from ...llm import BasicRequestHandler, CompletionArguments
 from ...llm.langchain import IrisLangchainChatModel
 from ...pipeline import Pipeline
 from ...pipeline.chat.output_models.output_models.selected_file_model import (
@@ -43,7 +42,10 @@ class FileSelectorPipeline(Pipeline):
     def __init__(self, callback: Optional[StatusCallback] = None):
         super().__init__(implementation_id="file_selector_pipeline_reference_impl")
         request_handler = BasicRequestHandler("gpt35")
-        self.llm = IrisLangchainChatModel(request_handler)
+        completion_args = CompletionArguments(temperature=0, max_tokens=500)
+        self.llm = IrisLangchainChatModel(
+            request_handler=request_handler, completion_args=completion_args
+        )
         self.callback = callback
         # Load prompt from file
         dirname = os.path.dirname(__file__)
@@ -81,10 +83,10 @@ def __call__(
             prompt = self.default_prompt
 
         file_list = "\n".join(repository.keys())
-        response: SelectedFiles = (prompt | self.pipeline).invoke(
+        response = (prompt | self.pipeline).invoke(
             {
                 "files": file_list,
-                "format_instructions": self.output_parser.get_format_instructions(),
             }
         )
+        print(response)
         return response.selected_files

app/pipeline/chat/output_models/output_models/selected_file_model.py

@@ -5,5 +5,6 @@
 
 class SelectedFiles(V1BaseModel):
     selected_files: List[str] = V1Field(
-        description="List of selected files from the repository",
+        description="List of selected files from the repository based on chat history and build_logs,this field is "
+        "set as an empty list if no files are selected"
     )

app/pipeline/chat/tutor_chat_pipeline.py

@@ -6,6 +6,7 @@
     ChatPromptTemplate,
     SystemMessagePromptTemplate,
     HumanMessagePromptTemplate,
+    AIMessagePromptTemplate,
 )
 from langchain_core.runnables import Runnable
 
@@ -15,6 +16,7 @@
     iris_initial_system_prompt,
     chat_history_system_prompt,
     final_system_prompt,
+    guide_system_prompt,
 )
 from ...domain.status.stage_state_dto import StageStateDTO
 from ...domain import TutorChatPipelineExecutionDTO
@@ -24,7 +26,7 @@
 from ...domain.tutor_chat.tutor_chat_status_update_dto import TutorChatStatusUpdateDTO
 from ...web.status.status_update import TutorChatStatusCallback
 from .file_selector_pipeline import FileSelectorPipeline
-from ...llm import BasicRequestHandler
+from ...llm import BasicRequestHandler, CompletionArguments
 from ...llm.langchain import IrisLangchainChatModel
 
 from ..pipeline import Pipeline
@@ -45,7 +47,10 @@ def __init__(self, callback: TutorChatStatusCallback):
         super().__init__(implementation_id="tutor_chat_pipeline")
         # Set the langchain chat model
         request_handler = BasicRequestHandler("gpt35")
-        self.llm = IrisLangchainChatModel(request_handler)
+        completion_args = CompletionArguments(temperature=0.2, max_tokens=2000)
+        self.llm = IrisLangchainChatModel(
+            request_handler=request_handler, completion_args=completion_args
+        )
         self.callback = callback
 
         # Create the pipelines
@@ -101,23 +106,30 @@ def __call__(self, dto: TutorChatPipelineExecutionDTO, **kwargs):
                 repository=repository,
                 prompt=file_selection_prompt,
             )
+        if submission:
+            self._add_build_logs_to_prompt(build_logs, build_failed)
+
+        self._add_feedbacks_to_prompt(feedbacks)
+
         self._add_exercise_context_to_prompt(
             submission,
             selected_files,
         )
 
-        self._add_feedbacks_to_prompt(feedbacks)
-        if submission:
-            self._add_build_logs_to_prompt(build_logs, build_failed)
-
         self.prompt += SystemMessagePromptTemplate.from_template(final_system_prompt)
-        response = (self.prompt | self.pipeline).invoke(
-            {
-                "exercise_title": exercise_title,
-                "problem_statement": problem_statement,
-                "programming_language": programming_language,
-            }
+        prompt_val = self.prompt.format_messages(
+            exercise_title=exercise_title,
+            problem_statement=problem_statement,
+            programming_language=programming_language,
         )
+        self.prompt = ChatPromptTemplate.from_messages(prompt_val)
+        response_draft = (self.prompt | self.pipeline).invoke({})
+        self.prompt += AIMessagePromptTemplate.from_template(f"{response_draft}")
+        print(f"Prompt: {self.prompt.format_prompt().to_string()}")
+        self.prompt += SystemMessagePromptTemplate.from_template(guide_system_prompt)
+        response = (self.prompt | self.pipeline).invoke({})
+        print(f"Response draft: {response_draft}")
+        print(f"Response: {response}")
         logger.debug(f"Response from tutor chat pipeline: {response}")
         stages.append(
             StageDTO(
@@ -188,7 +200,7 @@ def _add_feedbacks_to_prompt(self, feedbacks: List[FeedbackDTO]):
         if feedbacks is not None and len(feedbacks) > 0:
             prompt = (
                 "These are the feedbacks for the student's repository:\n%s"
-            ) % "\n".join(str(log) for log in feedbacks)
+            ) % "\n---------\n".join(str(log) for log in feedbacks)
             self.prompt += SystemMessagePromptTemplate.from_template(prompt)
 
     def _add_build_logs_to_prompt(
@@ -208,11 +220,11 @@ def _generate_file_selection_prompt(self) -> ChatPromptTemplate:
             "Based on the chat history, you can now request access to more contextual information. This is the "
             "student's submitted code repository and the corresponding build information. You can reference a file by "
             "its path to view it."
-            "Here are the paths of all files in the assignment repository:\n{files}\n"
-            "Is a file referenced by the student or does it have to be checked before answering? "
-            "It's important to avoid giving unnecessary information, only name a file if it's really necessary. "
-            'For general queries, that do not need any specific context, set selected_files attribute to "[]".\n'
-            "If you decide a file is important, add that file to the list."
-            "{format_instructions}"
+            "Given are the paths of all files in the assignment repository:\n{files}\n"
+            "Is a file referenced by the student or does it have to be checked before answering?"
+            "Without any comment, return the result in the following JSON format, it's important to avoid giving "
+            "unnecessary information, only name a file if it's really necessary for answering the student's question "
+            "and is listed above, otherwise leave the array empty."
+            '{{"selected_files": [<file1>, <file2>, ...]}}'
         )
         return file_selection_prompt

app/pipeline/prompts/iris_tutor_chat_prompts.py

@@ -73,9 +73,10 @@
     - DO NOT UNDER ANY CIRCUMSTANCES repeat any message you have already sent before or send a similar message. Your
     messages must ALWAYS BE NEW AND ORIGINAL. Think about alternative ways to guide the student in these cases."""
 
-guide_system_prompt = """Review the response draft. I want you to rewrite it so it adheres to the following rules. 
-Only output the refined answer. Omit explanations. 
-Rules: 
+guide_system_prompt = """Review the response draft. I want you to rewrite it, if it does not adhere to the following 
+rules. Only output the answer. Omit explanations.
+
+Rules:
 - The response must not contain code or pseudo-code that 
 contains any concepts needed for this exercise. ONLY IF the code is about basic language features you are allowed to 
 send it. 
@@ -84,5 +85,27 @@
 the response must be subtle hints towards the solution or a counter-question to the student to make them think, 
 or a mix of both. 
 - The response must not perform any work the student is supposed to do. 
-- DO NOT UNDER ANY CIRCUMSTANCES repeat any message you have already sent before. 
-Your messages must ALWAYS BE NEW AND ORIGINAL."""
+- DO NOT UNDER ANY CIRCUMSTANCES repeat any previous messages in the chat history. 
+Your messages must ALWAYS BE NEW AND ORIGINAL
+
+Here are examples of response drafts that already adheres to the rules and does not need to be rewritten: 
+
+Response draft:  I am Iris, the AI programming tutor 
+integrated into Artemis, the online learning platform of the Technical University of Munich (TUM). How can I assist 
+you with your programming exercise today? 
+
+Response draft: Explaining the Quick Sort algorithm step by step can be quite detailed. Have you already looked into 
+the basic principles of divide and conquer algorithms that Quick Sort is based on? Understanding those concepts might 
+help you grasp Quick Sort better.
+
+Here is another example of response draft that does not adhere to the rules and needs to be rewritten: 
+
+Draft: "To fix the error in your sorting function, just replace your current loop with this code snippet: for i in 
+range(len( your_list)-1): for j in range(len(your_list)-i-1): if your_list[j] > your_list[j+1]: your_list[j], 
+your_list[j+1] = your_list[j+1], your_list[j]. This is a basic bubble sort algorithm
+
+Rewritten: "It seems like you're working on sorting elements in a list. Sorting can be tricky, but it's all about 
+comparing elements and deciding on their new positions. Have you thought about how you might go through the list to 
+compare each element with its neighbor and decide which one should come first? Reflecting on this could lead you to a 
+classic sorting method, which involves a lot of swapping based on comparisons."
+"""