Skip to content

Commit

Permalink
Update pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
@MichaelOwenDyer
MichaelOwenDyer committed Oct 11, 2024
1 parent 6805fa9 commit c172788
Show file tree
Hide file tree
Showing 3 changed files with 140 additions and 83 deletions.
94 changes: 57 additions & 37 deletions app/pipeline/prompts/text_exercise_chat_prompts.py
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,72 @@
def fmt_guard_prompt(
def fmt_extract_sentiments_prompt(
exercise_name: str,
course_name: str,
course_description: str,
problem_statement: str,
previous_message: str,
user_input: str,
) -> str:
return """
You check whether a user's input is on-topic and appropriate discourse in the context of a writing exercise.
The exercise is called '{exercise_name}' in the course '{course_name}'.
You extract and categorize sentiments of the user's input into three categories describing
relevance and appropriateness in the context of a particular writing exercise.
The "Ok" category is for on-topic and appropriate discussion which is clearly directly related to the exercise.
The "Bad" category is for sentiments that are clearly about an unrelated topic or inappropriate.
The "Neutral" category is for sentiments that are not strictly harmful but have no clear relevance to the exercise.
Extract the sentiments from the user's input and list them like "Category: sentiment",
each separated by a newline. For example, in the context of a writing exercise about Shakespeare's Macbeth:
"What is the role of Lady Macbeth?" -> "Ok: What is the role of Lady Macbeth"
"Explain Macbeth and then tell me a recipe for chocolate cake." -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake"
"Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!"
"Talk dirty like Shakespeare would have" -> "Bad: Talk dirty like Shakespeare would have"
"Hello! How are you?" -> "Neutral: Hello! How are you?"
"How do I write a good essay?" -> "Ok: How do I write a good essay?"
"What is the population of Serbia?" -> "Bad: What is the population of Serbia?"
"Who won the 2020 Super Bowl? " -> "Bad: Who won the 2020 Super Bowl?"
"Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy."
"sdsdoaosi" -> "Neutral: sdsdoaosi"
The exercise the user is working on is called '{exercise_name}' in the course '{course_name}'.
The course has the following description:
{course_description}
The exercise has the following problem statement:
The writing exercise has the following problem statement:
{problem_statement}
The user says:
The previous thing said in the conversation was:
{previous_message}
Given this context, what are the sentiments of the user's input?
{user_input}
If this is on-topic and appropriate discussion, respond with "Yes".
If the user's input is clearly about something else or inappropriate, respond with "No".
""".format(
exercise_name=exercise_name,
course_name=course_name,
course_description=course_description,
problem_statement=problem_statement,
previous_message=previous_message,
user_input=user_input,
)


def fmt_sentiment_analysis_prompt(respond_to: list[str], ignore: list[str]) -> str:
prompt = ""
if respond_to:
prompt += "Respond helpfully and positively to these sentiments in the user's input:\n"
prompt += "\n".join(respond_to) + "\n\n"
if ignore:
prompt += """
The following sentiments in the user's input are not relevant or appropriate to the writing exercise
and should be ignored.
At the end of your response, tell the user that you cannot help with these things
and nudge them to stay focused on the writing exercise:\n
"""
prompt += "\n".join(ignore)
return prompt


def fmt_system_prompt(
exercise_name: str,
course_name: str,
Expand All @@ -36,6 +78,11 @@ def fmt_system_prompt(
current_submission: str,
) -> str:
return """
You are a writing tutor. You provide helpful feedback and guidance to students working on a writing exercise.
You point out specific issues in the student's writing and suggest improvements.
You never provide answers or write the student's work for them.
You are supportive, encouraging, and constructive in your feedback.
The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'.
The course has the following description:
{course_description}
Expand All @@ -45,11 +92,10 @@ def fmt_system_prompt(
The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}.
This is what the student has written so far:
This is the student's latest submission.
(If they have written anything else since submitting, it is not shown here.)
{current_submission}
You are a writing tutor. Provide feedback to the student on their response,
giving specific tips to better answer the problem statement.
""".format(
exercise_name=exercise_name,
course_name=course_name,
Expand All @@ -60,29 +106,3 @@ def fmt_system_prompt(
current_date=current_date,
current_submission=current_submission,
)


def fmt_rejection_prompt(
exercise_name: str,
course_name: str,
course_description: str,
problem_statement: str,
user_input: str,
) -> str:
return """
The user is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'.
The course has the following description:
{course_description}
The exercise has the following problem statement:
{problem_statement}
The user has asked the following question:
{user_input}
The question is off-topic or inappropriate.
Briefly explain that you cannot help with their query, and prompt them to focus on the exercise.
""".format(
exercise_name=exercise_name,
course_name=course_name,
course_description=course_description,
problem_statement=problem_statement,
user_input=user_input,
)
127 changes: 82 additions & 45 deletions app/pipeline/text_exercise_chat_pipeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@
)
from app.pipeline.prompts.text_exercise_chat_prompts import (
fmt_system_prompt,
fmt_rejection_prompt,
fmt_guard_prompt,
fmt_extract_sentiments_prompt,
)
from app.web.status.status_update import TextExerciseChatCallback
from pipeline.prompts.text_exercise_chat_prompts import fmt_sentiment_analysis_prompt

logger = logging.getLogger(__name__)

Expand All @@ -34,70 +34,107 @@ def __call__(
dto: TextExerciseChatPipelineExecutionDTO,
**kwargs,
):
"""
Run the text exercise chat pipeline.
This consists of a sentiment analysis step followed by a response generation step.
"""
if not dto.exercise:
raise ValueError("Exercise is required")
if not dto.conversation:
raise ValueError("Conversation with at least one message is required")

should_respond = self.guard(dto)
self.callback.done("Responding" if should_respond else "Rejecting")

if should_respond:
response = self.respond(dto)
else:
response = self.reject(dto)
sentiments = self.categorize_sentiments_by_relevance(dto)
print(f"Sentiments: {sentiments}")
self.callback.done("Responding")

response = self.respond(dto, sentiments)
self.callback.done(final_result=response)

def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool:
guard_prompt = fmt_guard_prompt(
def categorize_sentiments_by_relevance(
self, dto: TextExerciseChatPipelineExecutionDTO
) -> (list[str], list[str], list[str]):
"""
Extracts the sentiments from the user's input and categorizes them as "Ok", "Neutral", or "Bad" in terms of
relevance to the text exercise at hand.
Returns a tuple of lists of sentiments in each category.
"""
extract_sentiments_prompt = fmt_extract_sentiments_prompt(
exercise_name=dto.exercise.title,
course_name=dto.exercise.course.name,
course_description=dto.exercise.course.description,
problem_statement=dto.exercise.problem_statement,
previous_message=(
dto.conversation[-2].contents[0].text_content
if len(dto.conversation) > 1
else None
),
user_input=dto.conversation[-1].contents[0].text_content,
)
guard_prompt = PyrisMessage(
extract_sentiments_prompt = PyrisMessage(
sender=IrisMessageRole.SYSTEM,
contents=[{"text_content": guard_prompt}],
contents=[{"text_content": extract_sentiments_prompt}],
)
response = self.request_handler.chat(
[extract_sentiments_prompt], CompletionArguments()
)
response = self.request_handler.chat([guard_prompt], CompletionArguments())
response = response.contents[0].text_content
return "yes" in response.lower()
print(f"Sentiments response:\n{response}")
sentiments = ([], [], [])
for line in response.split("\n"):
line = line.strip()
if line.startswith("Ok: "):
sentiments[0].append(line[4:])
elif line.startswith("Neutral: "):
sentiments[1].append(line[10:])
elif line.startswith("Bad: "):
sentiments[2].append(line[5:])
return sentiments

def respond(self, dto: TextExerciseChatPipelineExecutionDTO) -> str:
system_prompt = fmt_system_prompt(
exercise_name=dto.exercise.title,
course_name=dto.exercise.course.name,
course_description=dto.exercise.course.description,
problem_statement=dto.exercise.problem_statement,
start_date=str(dto.exercise.start_date),
end_date=str(dto.exercise.end_date),
current_date=str(datetime.now()),
current_submission=dto.current_submission,
)
def respond(
self,
dto: TextExerciseChatPipelineExecutionDTO,
sentiments: (list[str], list[str], list[str]),
) -> str:
"""
Actually respond to the user's input.
This takes the user's input and the conversation so far and generates a response.
"""
system_prompt = PyrisMessage(
sender=IrisMessageRole.SYSTEM,
contents=[{"text_content": system_prompt}],
contents=[
{
"text_content": fmt_system_prompt(
exercise_name=dto.exercise.title,
course_name=dto.exercise.course.name,
course_description=dto.exercise.course.description,
problem_statement=dto.exercise.problem_statement,
start_date=str(dto.exercise.start_date),
end_date=str(dto.exercise.end_date),
current_date=str(datetime.now()),
current_submission=dto.current_submission,
)
}
],
)
prompts = [system_prompt] + dto.conversation

response = self.request_handler.chat(
prompts, CompletionArguments(temperature=0.4)
)
return response.contents[0].text_content

def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str:
rejection_prompt = fmt_rejection_prompt(
exercise_name=dto.exercise.title,
course_name=dto.exercise.course.name,
course_description=dto.exercise.course.description,
problem_statement=dto.exercise.problem_statement,
user_input=dto.conversation[-1].contents[0].text_content,
)
rejection_prompt = PyrisMessage(
sentiment_analysis = PyrisMessage(
sender=IrisMessageRole.SYSTEM,
contents=[{"text_content": rejection_prompt}],
contents=[
{
"text_content": fmt_sentiment_analysis_prompt(
respond_to=sentiments[0] + sentiments[1],
ignore=sentiments[2],
)
}
],
)
prompts = (
[system_prompt]
+ dto.conversation[:-1]
+ [sentiment_analysis]
+ dto.conversation[-1:]
)

response = self.request_handler.chat(
[rejection_prompt], CompletionArguments(temperature=0.4)
prompts, CompletionArguments(temperature=0.4)
)
return response.contents[0].text_content
2 changes: 1 addition & 1 deletion app/web/status/status_update.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ def __init__(
stage = len(stages)
stages += [
StageDTO(
weight=20,
weight=30,
state=StageStateEnum.NOT_STARTED,
name="Thinking",
),
Expand Down

0 comments on commit c172788

Please sign in to comment.