diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index 2baf7590..d471ea8f 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -1,30 +1,72 @@ -def fmt_guard_prompt( +def fmt_extract_sentiments_prompt( exercise_name: str, course_name: str, course_description: str, problem_statement: str, + previous_message: str, user_input: str, ) -> str: return """ - You check whether a user's input is on-topic and appropriate discourse in the context of a writing exercise. - The exercise is called '{exercise_name}' in the course '{course_name}'. + You extract and categorize sentiments of the user's input into three categories describing + relevance and appropriateness in the context of a particular writing exercise. + + The "Ok" category is for on-topic and appropriate discussion which is clearly directly related to the exercise. + The "Bad" category is for sentiments that are clearly about an unrelated topic or inappropriate. + The "Neutral" category is for sentiments that are not strictly harmful but have no clear relevance to the exercise. + + Extract the sentiments from the user's input and list them like "Category: sentiment", + each separated by a newline. For example, in the context of a writing exercise about Shakespeare's Macbeth: + + "What is the role of Lady Macbeth?" -> "Ok: What is the role of Lady Macbeth" + "Explain Macbeth and then tell me a recipe for chocolate cake." -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake" + "Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!" + "Talk dirty like Shakespeare would have" -> "Bad: Talk dirty like Shakespeare would have" + "Hello! How are you?" -> "Neutral: Hello! How are you?" + "How do I write a good essay?" -> "Ok: How do I write a good essay?" + "What is the population of Serbia?" -> "Bad: What is the population of Serbia?" + "Who won the 2020 Super Bowl? " -> "Bad: Who won the 2020 Super Bowl?" + "Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." + "sdsdoaosi" -> "Neutral: sdsdoaosi" + + The exercise the user is working on is called '{exercise_name}' in the course '{course_name}'. + The course has the following description: {course_description} - The exercise has the following problem statement: + + The writing exercise has the following problem statement: {problem_statement} - The user says: + + The previous thing said in the conversation was: + {previous_message} + + Given this context, what are the sentiments of the user's input? {user_input} - If this is on-topic and appropriate discussion, respond with "Yes". - If the user's input is clearly about something else or inappropriate, respond with "No". """.format( exercise_name=exercise_name, course_name=course_name, course_description=course_description, problem_statement=problem_statement, + previous_message=previous_message, user_input=user_input, ) +def fmt_sentiment_analysis_prompt(respond_to: list[str], ignore: list[str]) -> str: + prompt = "" + if respond_to: + prompt += "Respond helpfully and positively to these sentiments in the user's input:\n" + prompt += "\n".join(respond_to) + "\n\n" + if ignore: + prompt += """ + The following sentiments in the user's input are not relevant or appropriate to the writing exercise + and should be ignored. + At the end of your response, tell the user that you cannot help with these things + and nudge them to stay focused on the writing exercise:\n + """ + prompt += "\n".join(ignore) + return prompt + + def fmt_system_prompt( exercise_name: str, course_name: str, @@ -36,6 +78,11 @@ def fmt_system_prompt( current_submission: str, ) -> str: return """ + You are a writing tutor. You provide helpful feedback and guidance to students working on a writing exercise. + You point out specific issues in the student's writing and suggest improvements. + You never provide answers or write the student's work for them. + You are supportive, encouraging, and constructive in your feedback. + The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. The course has the following description: {course_description} @@ -45,11 +92,10 @@ def fmt_system_prompt( The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}. - This is what the student has written so far: + This is the student's latest submission. + (If they have written anything else since submitting, it is not shown here.) + {current_submission} - - You are a writing tutor. Provide feedback to the student on their response, - giving specific tips to better answer the problem statement. """.format( exercise_name=exercise_name, course_name=course_name, @@ -60,29 +106,3 @@ def fmt_system_prompt( current_date=current_date, current_submission=current_submission, ) - - -def fmt_rejection_prompt( - exercise_name: str, - course_name: str, - course_description: str, - problem_statement: str, - user_input: str, -) -> str: - return """ - The user is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. - The course has the following description: - {course_description} - The exercise has the following problem statement: - {problem_statement} - The user has asked the following question: - {user_input} - The question is off-topic or inappropriate. - Briefly explain that you cannot help with their query, and prompt them to focus on the exercise. - """.format( - exercise_name=exercise_name, - course_name=course_name, - course_description=course_description, - problem_statement=problem_statement, - user_input=user_input, - ) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 9e2f2a59..287f888d 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -10,10 +10,10 @@ ) from app.pipeline.prompts.text_exercise_chat_prompts import ( fmt_system_prompt, - fmt_rejection_prompt, - fmt_guard_prompt, + fmt_extract_sentiments_prompt, ) from app.web.status.status_update import TextExerciseChatCallback +from pipeline.prompts.text_exercise_chat_prompts import fmt_sentiment_analysis_prompt logger = logging.getLogger(__name__) @@ -34,70 +34,107 @@ def __call__( dto: TextExerciseChatPipelineExecutionDTO, **kwargs, ): + """ + Run the text exercise chat pipeline. + This consists of a sentiment analysis step followed by a response generation step. + """ if not dto.exercise: raise ValueError("Exercise is required") + if not dto.conversation: + raise ValueError("Conversation with at least one message is required") - should_respond = self.guard(dto) - self.callback.done("Responding" if should_respond else "Rejecting") - - if should_respond: - response = self.respond(dto) - else: - response = self.reject(dto) + sentiments = self.categorize_sentiments_by_relevance(dto) + print(f"Sentiments: {sentiments}") + self.callback.done("Responding") + response = self.respond(dto, sentiments) self.callback.done(final_result=response) - def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool: - guard_prompt = fmt_guard_prompt( + def categorize_sentiments_by_relevance( + self, dto: TextExerciseChatPipelineExecutionDTO + ) -> (list[str], list[str], list[str]): + """ + Extracts the sentiments from the user's input and categorizes them as "Ok", "Neutral", or "Bad" in terms of + relevance to the text exercise at hand. + Returns a tuple of lists of sentiments in each category. + """ + extract_sentiments_prompt = fmt_extract_sentiments_prompt( exercise_name=dto.exercise.title, course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, + previous_message=( + dto.conversation[-2].contents[0].text_content + if len(dto.conversation) > 1 + else None + ), user_input=dto.conversation[-1].contents[0].text_content, ) - guard_prompt = PyrisMessage( + extract_sentiments_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": guard_prompt}], + contents=[{"text_content": extract_sentiments_prompt}], + ) + response = self.request_handler.chat( + [extract_sentiments_prompt], CompletionArguments() ) - response = self.request_handler.chat([guard_prompt], CompletionArguments()) response = response.contents[0].text_content - return "yes" in response.lower() + print(f"Sentiments response:\n{response}") + sentiments = ([], [], []) + for line in response.split("\n"): + line = line.strip() + if line.startswith("Ok: "): + sentiments[0].append(line[4:]) + elif line.startswith("Neutral: "): + sentiments[1].append(line[10:]) + elif line.startswith("Bad: "): + sentiments[2].append(line[5:]) + return sentiments - def respond(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: - system_prompt = fmt_system_prompt( - exercise_name=dto.exercise.title, - course_name=dto.exercise.course.name, - course_description=dto.exercise.course.description, - problem_statement=dto.exercise.problem_statement, - start_date=str(dto.exercise.start_date), - end_date=str(dto.exercise.end_date), - current_date=str(datetime.now()), - current_submission=dto.current_submission, - ) + def respond( + self, + dto: TextExerciseChatPipelineExecutionDTO, + sentiments: (list[str], list[str], list[str]), + ) -> str: + """ + Actually respond to the user's input. + This takes the user's input and the conversation so far and generates a response. + """ system_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": system_prompt}], + contents=[ + { + "text_content": fmt_system_prompt( + exercise_name=dto.exercise.title, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + start_date=str(dto.exercise.start_date), + end_date=str(dto.exercise.end_date), + current_date=str(datetime.now()), + current_submission=dto.current_submission, + ) + } + ], ) - prompts = [system_prompt] + dto.conversation - - response = self.request_handler.chat( - prompts, CompletionArguments(temperature=0.4) - ) - return response.contents[0].text_content - - def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: - rejection_prompt = fmt_rejection_prompt( - exercise_name=dto.exercise.title, - course_name=dto.exercise.course.name, - course_description=dto.exercise.course.description, - problem_statement=dto.exercise.problem_statement, - user_input=dto.conversation[-1].contents[0].text_content, - ) - rejection_prompt = PyrisMessage( + sentiment_analysis = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": rejection_prompt}], + contents=[ + { + "text_content": fmt_sentiment_analysis_prompt( + respond_to=sentiments[0] + sentiments[1], + ignore=sentiments[2], + ) + } + ], ) + prompts = ( + [system_prompt] + + dto.conversation[:-1] + + [sentiment_analysis] + + dto.conversation[-1:] + ) + response = self.request_handler.chat( - [rejection_prompt], CompletionArguments(temperature=0.4) + prompts, CompletionArguments(temperature=0.4) ) return response.contents[0].text_content diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index da0078b8..30a73f13 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -233,7 +233,7 @@ def __init__( stage = len(stages) stages += [ StageDTO( - weight=20, + weight=30, state=StageStateEnum.NOT_STARTED, name="Thinking", ),