From 4e5e5387f45ed0cfb9a168b40aeb30d802b45db2 Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Sun, 25 Aug 2024 23:58:56 +0200 Subject: [PATCH 01/20] Support settings V3 --- app/domain/__init__.py | 1 + app/domain/feature_dto.py | 7 ++++++ app/web/routers/pipelines.py | 49 +++++++++++++++++++++++++++++++++--- 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 app/domain/feature_dto.py diff --git a/app/domain/__init__.py b/app/domain/__init__.py index 2f56f3f3..837b0741 100644 --- a/app/domain/__init__.py +++ b/app/domain/__init__.py @@ -11,3 +11,4 @@ ) from .pyris_message import PyrisMessage, IrisMessageRole from app.domain.data import image_message_content_dto +from app.domain.feature_dto import FeatureDTO diff --git a/app/domain/feature_dto.py b/app/domain/feature_dto.py new file mode 100644 index 00000000..997e2722 --- /dev/null +++ b/app/domain/feature_dto.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel + + +class FeatureDTO(BaseModel): + id: str + name: str + description: str diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index 7ac9d3da..d6cae112 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -17,6 +17,7 @@ from app.pipeline.chat.course_chat_pipeline import CourseChatPipeline from app.pipeline.chat.exercise_chat_pipeline import ExerciseChatPipeline from app.dependencies import TokenValidator +from app.domain import FeatureDTO router = APIRouter(prefix="/api/v1/pipelines", tags=["pipelines"]) logger = logging.getLogger(__name__) @@ -86,9 +87,51 @@ def run_course_chat_pipeline(variant: str, dto: CourseChatPipelineExecutionDTO): thread.start() -@router.get("/{feature}") +@router.get("/{feature}/variants") def get_pipeline(feature: str): """ - Get the pipeline for the given feature. + Get the pipeline variants for the given feature. """ - return Response(status_code=status.HTTP_501_NOT_IMPLEMENTED) + match feature: + case "CHAT": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default chat variant.", + ) + ] + case "PROGRAMMING_EXERCISE_CHAT": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default programming exercise chat variant.", + ) + ] + case "COURSE_CHAT": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default course chat variant.", + ) + ] + case "COMPETENCY_GENERATION": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default competency generation variant.", + ) + ] + case "LECTURE_INGESTION": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default lecture ingestion variant.", + ) + ] + case _: + return Response(status_code=status.HTTP_400_BAD_REQUEST) From 6c79346b79971b9b82cbb2ea642ed5c250a54b90 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Tue, 3 Sep 2024 17:21:21 +0200 Subject: [PATCH 02/20] Initial commit --- app/domain/data/text_exercise_dto.py | 15 +++++ ...xt_exercise_chat_pipeline_execution_dto.py | 10 ++++ .../prompts/text_exercise_chat_prompts.py | 35 +++++++++++ app/pipeline/text_exercise_chat_pipeline.py | 59 +++++++++++++++++++ app/web/routers/pipelines.py | 43 ++++++++++++++ app/web/status/status_update.py | 22 +++++++ 6 files changed, 184 insertions(+) create mode 100644 app/domain/data/text_exercise_dto.py create mode 100644 app/domain/text_exercise_chat_pipeline_execution_dto.py create mode 100644 app/pipeline/prompts/text_exercise_chat_prompts.py create mode 100644 app/pipeline/text_exercise_chat_pipeline.py diff --git a/app/domain/data/text_exercise_dto.py b/app/domain/data/text_exercise_dto.py new file mode 100644 index 00000000..7040b181 --- /dev/null +++ b/app/domain/data/text_exercise_dto.py @@ -0,0 +1,15 @@ +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel, Field + +from domain.data.course_dto import CourseDTO + + +class TextExerciseDTO(BaseModel): + id: int + name: str + course: CourseDTO + problem_statement: str = Field(alias="problemStatement") + start_date: Optional[datetime] = Field(alias="startDate", default=None) + end_date: Optional[datetime] = Field(alias="endDate", default=None) diff --git a/app/domain/text_exercise_chat_pipeline_execution_dto.py b/app/domain/text_exercise_chat_pipeline_execution_dto.py new file mode 100644 index 00000000..03ff7c19 --- /dev/null +++ b/app/domain/text_exercise_chat_pipeline_execution_dto.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel, Field + +from domain import PipelineExecutionDTO +from domain.data.text_exercise_dto import TextExerciseDTO + + +class TextExerciseChatPipelineExecutionDTO(BaseModel): + execution: PipelineExecutionDTO + exercise: TextExerciseDTO + current_answer: str = Field(alias="currentAnswer") diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py new file mode 100644 index 00000000..390cb954 --- /dev/null +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -0,0 +1,35 @@ +def system_prompt( + exercise_name: str, + course_name: str, + course_description: str, + problem_statement: str, + start_date: str, + end_date: str, + current_date: str, + current_answer: str, +) -> str: + return """ + The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. + The course has the following description: + {course_description} + + The exercise has the following problem statement: + {problem_statement} + + The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}. + + This is what the student has written so far: + {current_answer} + + You are a writing tutor. Provide feedback to the student on their response, + giving specific tips to better answer the problem statement. + """.format( + exercise_name=exercise_name, + course_name=course_name, + course_description=course_description, + problem_statement=problem_statement, + start_date=start_date, + end_date=end_date, + current_date=current_date, + current_answer=current_answer, + ) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py new file mode 100644 index 00000000..253504a2 --- /dev/null +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -0,0 +1,59 @@ +import logging +from datetime import datetime +from typing import Optional + +from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments +from app.pipeline import Pipeline +from domain import PyrisMessage, IrisMessageRole +from domain.data.text_message_content_dto import TextMessageContentDTO +from domain.text_exercise_chat_pipeline_execution_dto import ( + TextExerciseChatPipelineExecutionDTO, +) +from pipeline.prompts.text_exercise_chat_prompts import system_prompt +from web.status.status_update import TextExerciseChatCallback + +logger = logging.getLogger(__name__) + + +class TextExerciseChatPipeline(Pipeline): + callback: TextExerciseChatCallback + request_handler: CapabilityRequestHandler + + def __init__(self, callback: Optional[TextExerciseChatCallback] = None): + super().__init__(implementation_id="text_exercise_chat_pipeline_reference_impl") + self.callback = callback + self.request_handler = CapabilityRequestHandler( + requirements=RequirementList(context_length=8000) + ) + + def __call__( + self, + dto: TextExerciseChatPipelineExecutionDTO, + **kwargs, + ): + if not dto.exercise: + raise ValueError("Exercise is required") + + prompt = system_prompt( + exercise_name=dto.exercise.name, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + start_date=str(dto.exercise.start_date), + end_date=str(dto.exercise.end_date), + current_date=str(datetime.now()), + current_answer=dto.current_answer, + ) + prompt = PyrisMessage( + sender=IrisMessageRole.SYSTEM, + contents=[TextMessageContentDTO(text_content=prompt)], + ) + + # done building prompt + + response = self.request_handler.chat( + [prompt], CompletionArguments(temperature=0.4) + ) + response = response.contents[0].text_content + + self.callback.done(response) diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index eb198199..fae60926 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -21,6 +21,11 @@ from app.dependencies import TokenValidator from app.domain import FeatureDTO from app.pipeline.competency_extraction_pipeline import CompetencyExtractionPipeline +from domain.text_exercise_chat_pipeline_execution_dto import ( + TextExerciseChatPipelineExecutionDTO, +) +from pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline +from web.status.status_update import TextExerciseChatCallback router = APIRouter(prefix="/api/v1/pipelines", tags=["pipelines"]) logger = logging.getLogger(__name__) @@ -90,6 +95,44 @@ def run_course_chat_pipeline(variant: str, dto: CourseChatPipelineExecutionDTO): thread.start() +def run_text_exercise_chat_pipeline_worker(dto, variant): + try: + callback = TextExerciseChatCallback( + run_id=dto.settings.authentication_token, + base_url=dto.settings.artemis_base_url, + initial_stages=dto.initial_stages, + ) + match variant: + case "default" | "text_exercise_chat_pipeline_reference_impl": + pipeline = TextExerciseChatPipeline(callback=callback) + case _: + raise ValueError(f"Unknown variant: {variant}") + except Exception as e: + logger.error(f"Error preparing text exercise chat pipeline: {e}") + logger.error(traceback.format_exc()) + capture_exception(e) + return + + try: + pipeline(dto=dto) + except Exception as e: + logger.error(f"Error running text exercise chat pipeline: {e}") + logger.error(traceback.format_exc()) + callback.error("Fatal error.", exception=e) + + +@router.post( + "/text-exercise-chat/{variant}/run", + status_code=status.HTTP_202_ACCEPTED, + dependencies=[Depends(TokenValidator())], +) +def run_text_exercise_chat_pipeline( + variant: str, dto: TextExerciseChatPipelineExecutionDTO +): + thread = Thread(target=run_text_exercise_chat_pipeline_worker, args=(dto, variant)) + thread.start() + + def run_competency_extraction_pipeline_worker( dto: CompetencyExtractionPipelineExecutionDTO, _variant: str ): diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index 1f497f75..1ddf1ca9 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -218,6 +218,28 @@ def __init__( super().__init__(url, run_id, status, stage, current_stage_index) +class TextExerciseChatCallback(StatusCallback): + def __init__( + self, + run_id: str, + base_url: str, + initial_stages: List[StageDTO], + ): + url = f"{base_url}/api/public/pyris/pipelines/text-exercise-chat/runs/{run_id}/status" + stages = initial_stages or [] + stage = len(stages) + stages += [ + StageDTO( + weight=40, + state=StageStateEnum.NOT_STARTED, + name="Thinking", + ) + ] + super().__init__( + url, run_id, StatusUpdateDTO(stages=stages), stages[stage], stage + ) + + class CompetencyExtractionCallback(StatusCallback): def __init__( self, From 52d6bf8b27b9f76d03c14dd4c59ab412d39b94e8 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Sun, 22 Sep 2024 19:19:12 +0200 Subject: [PATCH 03/20] Implement text exercise chat pipeline --- app/domain/data/text_exercise_dto.py | 2 +- .../text_exercise_chat_status_update_dto.py | 5 ++ ...xt_exercise_chat_pipeline_execution_dto.py | 5 +- .../prompts/text_exercise_chat_prompts.py | 60 ++++++++++++++-- app/pipeline/text_exercise_chat_pipeline.py | 68 +++++++++++++++---- app/web/routers/pipelines.py | 14 +++- app/web/status/status_update.py | 30 +++++--- 7 files changed, 153 insertions(+), 31 deletions(-) create mode 100644 app/domain/status/text_exercise_chat_status_update_dto.py diff --git a/app/domain/data/text_exercise_dto.py b/app/domain/data/text_exercise_dto.py index 7040b181..3fd15330 100644 --- a/app/domain/data/text_exercise_dto.py +++ b/app/domain/data/text_exercise_dto.py @@ -8,7 +8,7 @@ class TextExerciseDTO(BaseModel): id: int - name: str + title: str course: CourseDTO problem_statement: str = Field(alias="problemStatement") start_date: Optional[datetime] = Field(alias="startDate", default=None) diff --git a/app/domain/status/text_exercise_chat_status_update_dto.py b/app/domain/status/text_exercise_chat_status_update_dto.py new file mode 100644 index 00000000..dd063ff7 --- /dev/null +++ b/app/domain/status/text_exercise_chat_status_update_dto.py @@ -0,0 +1,5 @@ +from app.domain.status.status_update_dto import StatusUpdateDTO + + +class TextExerciseChatStatusUpdateDTO(StatusUpdateDTO): + result: str = [] diff --git a/app/domain/text_exercise_chat_pipeline_execution_dto.py b/app/domain/text_exercise_chat_pipeline_execution_dto.py index 03ff7c19..efae1adf 100644 --- a/app/domain/text_exercise_chat_pipeline_execution_dto.py +++ b/app/domain/text_exercise_chat_pipeline_execution_dto.py @@ -1,10 +1,11 @@ from pydantic import BaseModel, Field -from domain import PipelineExecutionDTO +from domain import PipelineExecutionDTO, PyrisMessage from domain.data.text_exercise_dto import TextExerciseDTO class TextExerciseChatPipelineExecutionDTO(BaseModel): execution: PipelineExecutionDTO exercise: TextExerciseDTO - current_answer: str = Field(alias="currentAnswer") + conversation: list[PyrisMessage] = Field(default=[]) + current_submission: str = Field(alias="currentSubmission", default="") diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index 390cb954..cf499241 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -1,4 +1,30 @@ -def system_prompt( +def fmt_guard_prompt( + exercise_name: str, + course_name: str, + course_description: str, + problem_statement: str, + user_input: str, +) -> str: + return """ + You check whether a user's input is on-topic and appropriate discourse in the context of a writing exercise. + The exercise is called '{exercise_name}' in the course '{course_name}'. + The course has the following description: + {course_description} + The exercise has the following problem statement: + {problem_statement} + The user says: + {user_input} + If this is on-topic and appropriate discussion, respond with "Yes". If not, respond with "No". + """.format( + exercise_name=exercise_name, + course_name=course_name, + course_description=course_description, + problem_statement=problem_statement, + user_input=user_input, + ) + + +def fmt_system_prompt( exercise_name: str, course_name: str, course_description: str, @@ -6,7 +32,7 @@ def system_prompt( start_date: str, end_date: str, current_date: str, - current_answer: str, + current_submission: str, ) -> str: return """ The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. @@ -19,7 +45,7 @@ def system_prompt( The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}. This is what the student has written so far: - {current_answer} + {current_submission} You are a writing tutor. Provide feedback to the student on their response, giving specific tips to better answer the problem statement. @@ -31,5 +57,31 @@ def system_prompt( start_date=start_date, end_date=end_date, current_date=current_date, - current_answer=current_answer, + current_submission=current_submission, + ) + + +def fmt_rejection_prompt( + exercise_name: str, + course_name: str, + course_description: str, + problem_statement: str, + user_input: str, +) -> str: + return """ + The user is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. + The course has the following description: + {course_description} + The exercise has the following problem statement: + {problem_statement} + The user has asked the following question: + {user_input} + The question is off-topic or inappropriate. + Briefly explain that you cannot help with their query, and prompt them to focus on the exercise. + """.format( + exercise_name=exercise_name, + course_name=course_name, + course_description=course_description, + problem_statement=problem_statement, + user_input=user_input, ) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 253504a2..38d5754d 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -5,11 +5,14 @@ from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments from app.pipeline import Pipeline from domain import PyrisMessage, IrisMessageRole -from domain.data.text_message_content_dto import TextMessageContentDTO from domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) -from pipeline.prompts.text_exercise_chat_prompts import system_prompt +from pipeline.prompts.text_exercise_chat_prompts import ( + fmt_system_prompt, + fmt_rejection_prompt, + fmt_guard_prompt, +) from web.status.status_update import TextExerciseChatCallback logger = logging.getLogger(__name__) @@ -34,26 +37,67 @@ def __call__( if not dto.exercise: raise ValueError("Exercise is required") - prompt = system_prompt( - exercise_name=dto.exercise.name, + should_respond = self.guard(dto) + self.callback.done("Responding" if should_respond else "Rejecting") + + if should_respond: + response = self.respond(dto) + else: + response = self.reject(dto) + + self.callback.done(final_result=response) + + def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool: + guard_prompt = fmt_guard_prompt( + exercise_name=dto.exercise.title, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + user_input=dto.current_submission, + ) + guard_prompt = PyrisMessage( + sender=IrisMessageRole.SYSTEM, + contents=[{"text_content": guard_prompt}], + ) + response = self.request_handler.chat([guard_prompt], CompletionArguments()) + response = response.contents[0].text_content + return "yes" in response.lower() + + def respond(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: + system_prompt = fmt_system_prompt( + exercise_name=dto.exercise.title, course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, start_date=str(dto.exercise.start_date), end_date=str(dto.exercise.end_date), current_date=str(datetime.now()), - current_answer=dto.current_answer, + current_submission=dto.current_submission, ) - prompt = PyrisMessage( + system_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[TextMessageContentDTO(text_content=prompt)], + contents=[{"text_content": system_prompt}], ) - - # done building prompt + prompts = [system_prompt] + dto.conversation response = self.request_handler.chat( - [prompt], CompletionArguments(temperature=0.4) + prompts, CompletionArguments(temperature=0.4) ) - response = response.contents[0].text_content + return response.contents[0].text_content - self.callback.done(response) + def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: + rejection_prompt = fmt_rejection_prompt( + exercise_name=dto.exercise.title, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + user_input=dto.current_submission, + ) + rejection_prompt = PyrisMessage( + sender=IrisMessageRole.SYSTEM, + contents=[{"text_content": rejection_prompt}], + ) + response = self.request_handler.chat( + [rejection_prompt], CompletionArguments(temperature=0.4) + ) + return response.contents[0].text_content diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index fae60926..c53ab668 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -98,9 +98,9 @@ def run_course_chat_pipeline(variant: str, dto: CourseChatPipelineExecutionDTO): def run_text_exercise_chat_pipeline_worker(dto, variant): try: callback = TextExerciseChatCallback( - run_id=dto.settings.authentication_token, - base_url=dto.settings.artemis_base_url, - initial_stages=dto.initial_stages, + run_id=dto.execution.settings.authentication_token, + base_url=dto.execution.settings.artemis_base_url, + initial_stages=dto.execution.initial_stages, ) match variant: case "default" | "text_exercise_chat_pipeline_reference_impl": @@ -193,6 +193,14 @@ def get_pipeline(feature: str): description="Default programming exercise chat variant.", ) ] + case "TEXT_EXERCISE_CHAT": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default text exercise chat variant.", + ) + ] case "COURSE_CHAT": return [ FeatureDTO( diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index 1ddf1ca9..da0078b8 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -5,18 +5,21 @@ import requests from abc import ABC -from ...domain.status.competency_extraction_status_update_dto import ( +from app.domain.status.competency_extraction_status_update_dto import ( CompetencyExtractionStatusUpdateDTO, ) -from ...domain.chat.course_chat.course_chat_status_update_dto import ( +from app.domain.chat.course_chat.course_chat_status_update_dto import ( CourseChatStatusUpdateDTO, ) -from ...domain.status.stage_state_dto import StageStateEnum -from ...domain.status.stage_dto import StageDTO -from ...domain.chat.exercise_chat.exercise_chat_status_update_dto import ( +from app.domain.status.stage_state_dto import StageStateEnum +from app.domain.status.stage_dto import StageDTO +from app.domain.status.text_exercise_chat_status_update_dto import ( + TextExerciseChatStatusUpdateDTO, +) +from app.domain.chat.exercise_chat.exercise_chat_status_update_dto import ( ExerciseChatStatusUpdateDTO, ) -from ...domain.status.status_update_dto import StatusUpdateDTO +from app.domain.status.status_update_dto import StatusUpdateDTO import logging logger = logging.getLogger(__name__) @@ -230,13 +233,22 @@ def __init__( stage = len(stages) stages += [ StageDTO( - weight=40, + weight=20, state=StageStateEnum.NOT_STARTED, name="Thinking", - ) + ), + StageDTO( + weight=20, + state=StageStateEnum.NOT_STARTED, + name="Responding", + ), ] super().__init__( - url, run_id, StatusUpdateDTO(stages=stages), stages[stage], stage + url, + run_id, + TextExerciseChatStatusUpdateDTO(stages=stages), + stages[stage], + stage, ) From ff3f259c0652baebf2d94afba86512f31eb96d29 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 18:33:50 +0200 Subject: [PATCH 04/20] Fix imports --- app/domain/data/text_exercise_dto.py | 2 +- app/domain/text_exercise_chat_pipeline_execution_dto.py | 4 ++-- app/pipeline/text_exercise_chat_pipeline.py | 4 ++-- app/web/routers/pipelines.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/domain/data/text_exercise_dto.py b/app/domain/data/text_exercise_dto.py index 3fd15330..098779f3 100644 --- a/app/domain/data/text_exercise_dto.py +++ b/app/domain/data/text_exercise_dto.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field -from domain.data.course_dto import CourseDTO +from app.domain.data.course_dto import CourseDTO class TextExerciseDTO(BaseModel): diff --git a/app/domain/text_exercise_chat_pipeline_execution_dto.py b/app/domain/text_exercise_chat_pipeline_execution_dto.py index efae1adf..65e8871c 100644 --- a/app/domain/text_exercise_chat_pipeline_execution_dto.py +++ b/app/domain/text_exercise_chat_pipeline_execution_dto.py @@ -1,7 +1,7 @@ from pydantic import BaseModel, Field -from domain import PipelineExecutionDTO, PyrisMessage -from domain.data.text_exercise_dto import TextExerciseDTO +from app.domain import PipelineExecutionDTO, PyrisMessage +from app.domain.data.text_exercise_dto import TextExerciseDTO class TextExerciseChatPipelineExecutionDTO(BaseModel): diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 38d5754d..5fc6cca6 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -4,8 +4,8 @@ from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments from app.pipeline import Pipeline -from domain import PyrisMessage, IrisMessageRole -from domain.text_exercise_chat_pipeline_execution_dto import ( +from app.domain import PyrisMessage, IrisMessageRole +from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) from pipeline.prompts.text_exercise_chat_prompts import ( diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index c53ab668..92171ed3 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -21,7 +21,7 @@ from app.dependencies import TokenValidator from app.domain import FeatureDTO from app.pipeline.competency_extraction_pipeline import CompetencyExtractionPipeline -from domain.text_exercise_chat_pipeline_execution_dto import ( +from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) from pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline From 72067852ba42933d9cbcae534d5dcc38d999d8d2 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 18:47:31 +0200 Subject: [PATCH 05/20] Fix imports --- app/pipeline/text_exercise_chat_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 5fc6cca6..3aad4625 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -8,12 +8,12 @@ from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) -from pipeline.prompts.text_exercise_chat_prompts import ( +from app.pipeline.prompts.text_exercise_chat_prompts import ( fmt_system_prompt, fmt_rejection_prompt, fmt_guard_prompt, ) -from web.status.status_update import TextExerciseChatCallback +from app.web.status.status_update import TextExerciseChatCallback logger = logging.getLogger(__name__) From 80a846cd5838d4b99cec58fec8ba5ceba88c873e Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 18:54:49 +0200 Subject: [PATCH 06/20] Fix imports --- app/web/routers/pipelines.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index 92171ed3..7bc047c0 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -24,8 +24,8 @@ from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) -from pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline -from web.status.status_update import TextExerciseChatCallback +from app.pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline +from app.web.status.status_update import TextExerciseChatCallback router = APIRouter(prefix="/api/v1/pipelines", tags=["pipelines"]) logger = logging.getLogger(__name__) From 4042ab52e99367ef70eb1527522596192493b512 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 19:12:37 +0200 Subject: [PATCH 07/20] Fix --- app/pipeline/prompts/text_exercise_chat_prompts.py | 3 ++- app/pipeline/text_exercise_chat_pipeline.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index cf499241..2baf7590 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -14,7 +14,8 @@ def fmt_guard_prompt( {problem_statement} The user says: {user_input} - If this is on-topic and appropriate discussion, respond with "Yes". If not, respond with "No". + If this is on-topic and appropriate discussion, respond with "Yes". + If the user's input is clearly about something else or inappropriate, respond with "No". """.format( exercise_name=exercise_name, course_name=course_name, diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 3aad4625..9e2f2a59 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -53,7 +53,7 @@ def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool: course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, - user_input=dto.current_submission, + user_input=dto.conversation[-1].contents[0].text_content, ) guard_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, @@ -91,7 +91,7 @@ def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, - user_input=dto.current_submission, + user_input=dto.conversation[-1].contents[0].text_content, ) rejection_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, From ed4d5dd7ea750206ca80c7128ee6155f8f0279d5 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Thu, 10 Oct 2024 13:33:20 +0200 Subject: [PATCH 08/20] Update pipeline --- .../prompts/text_exercise_chat_prompts.py | 94 ++++++++----- app/pipeline/text_exercise_chat_pipeline.py | 127 +++++++++++------- app/web/status/status_update.py | 2 +- 3 files changed, 140 insertions(+), 83 deletions(-) diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index 2baf7590..d471ea8f 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -1,30 +1,72 @@ -def fmt_guard_prompt( +def fmt_extract_sentiments_prompt( exercise_name: str, course_name: str, course_description: str, problem_statement: str, + previous_message: str, user_input: str, ) -> str: return """ - You check whether a user's input is on-topic and appropriate discourse in the context of a writing exercise. - The exercise is called '{exercise_name}' in the course '{course_name}'. + You extract and categorize sentiments of the user's input into three categories describing + relevance and appropriateness in the context of a particular writing exercise. + + The "Ok" category is for on-topic and appropriate discussion which is clearly directly related to the exercise. + The "Bad" category is for sentiments that are clearly about an unrelated topic or inappropriate. + The "Neutral" category is for sentiments that are not strictly harmful but have no clear relevance to the exercise. + + Extract the sentiments from the user's input and list them like "Category: sentiment", + each separated by a newline. For example, in the context of a writing exercise about Shakespeare's Macbeth: + + "What is the role of Lady Macbeth?" -> "Ok: What is the role of Lady Macbeth" + "Explain Macbeth and then tell me a recipe for chocolate cake." -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake" + "Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!" + "Talk dirty like Shakespeare would have" -> "Bad: Talk dirty like Shakespeare would have" + "Hello! How are you?" -> "Neutral: Hello! How are you?" + "How do I write a good essay?" -> "Ok: How do I write a good essay?" + "What is the population of Serbia?" -> "Bad: What is the population of Serbia?" + "Who won the 2020 Super Bowl? " -> "Bad: Who won the 2020 Super Bowl?" + "Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." + "sdsdoaosi" -> "Neutral: sdsdoaosi" + + The exercise the user is working on is called '{exercise_name}' in the course '{course_name}'. + The course has the following description: {course_description} - The exercise has the following problem statement: + + The writing exercise has the following problem statement: {problem_statement} - The user says: + + The previous thing said in the conversation was: + {previous_message} + + Given this context, what are the sentiments of the user's input? {user_input} - If this is on-topic and appropriate discussion, respond with "Yes". - If the user's input is clearly about something else or inappropriate, respond with "No". """.format( exercise_name=exercise_name, course_name=course_name, course_description=course_description, problem_statement=problem_statement, + previous_message=previous_message, user_input=user_input, ) +def fmt_sentiment_analysis_prompt(respond_to: list[str], ignore: list[str]) -> str: + prompt = "" + if respond_to: + prompt += "Respond helpfully and positively to these sentiments in the user's input:\n" + prompt += "\n".join(respond_to) + "\n\n" + if ignore: + prompt += """ + The following sentiments in the user's input are not relevant or appropriate to the writing exercise + and should be ignored. + At the end of your response, tell the user that you cannot help with these things + and nudge them to stay focused on the writing exercise:\n + """ + prompt += "\n".join(ignore) + return prompt + + def fmt_system_prompt( exercise_name: str, course_name: str, @@ -36,6 +78,11 @@ def fmt_system_prompt( current_submission: str, ) -> str: return """ + You are a writing tutor. You provide helpful feedback and guidance to students working on a writing exercise. + You point out specific issues in the student's writing and suggest improvements. + You never provide answers or write the student's work for them. + You are supportive, encouraging, and constructive in your feedback. + The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. The course has the following description: {course_description} @@ -45,11 +92,10 @@ def fmt_system_prompt( The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}. - This is what the student has written so far: + This is the student's latest submission. + (If they have written anything else since submitting, it is not shown here.) + {current_submission} - - You are a writing tutor. Provide feedback to the student on their response, - giving specific tips to better answer the problem statement. """.format( exercise_name=exercise_name, course_name=course_name, @@ -60,29 +106,3 @@ def fmt_system_prompt( current_date=current_date, current_submission=current_submission, ) - - -def fmt_rejection_prompt( - exercise_name: str, - course_name: str, - course_description: str, - problem_statement: str, - user_input: str, -) -> str: - return """ - The user is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. - The course has the following description: - {course_description} - The exercise has the following problem statement: - {problem_statement} - The user has asked the following question: - {user_input} - The question is off-topic or inappropriate. - Briefly explain that you cannot help with their query, and prompt them to focus on the exercise. - """.format( - exercise_name=exercise_name, - course_name=course_name, - course_description=course_description, - problem_statement=problem_statement, - user_input=user_input, - ) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 9e2f2a59..287f888d 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -10,10 +10,10 @@ ) from app.pipeline.prompts.text_exercise_chat_prompts import ( fmt_system_prompt, - fmt_rejection_prompt, - fmt_guard_prompt, + fmt_extract_sentiments_prompt, ) from app.web.status.status_update import TextExerciseChatCallback +from pipeline.prompts.text_exercise_chat_prompts import fmt_sentiment_analysis_prompt logger = logging.getLogger(__name__) @@ -34,70 +34,107 @@ def __call__( dto: TextExerciseChatPipelineExecutionDTO, **kwargs, ): + """ + Run the text exercise chat pipeline. + This consists of a sentiment analysis step followed by a response generation step. + """ if not dto.exercise: raise ValueError("Exercise is required") + if not dto.conversation: + raise ValueError("Conversation with at least one message is required") - should_respond = self.guard(dto) - self.callback.done("Responding" if should_respond else "Rejecting") - - if should_respond: - response = self.respond(dto) - else: - response = self.reject(dto) + sentiments = self.categorize_sentiments_by_relevance(dto) + print(f"Sentiments: {sentiments}") + self.callback.done("Responding") + response = self.respond(dto, sentiments) self.callback.done(final_result=response) - def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool: - guard_prompt = fmt_guard_prompt( + def categorize_sentiments_by_relevance( + self, dto: TextExerciseChatPipelineExecutionDTO + ) -> (list[str], list[str], list[str]): + """ + Extracts the sentiments from the user's input and categorizes them as "Ok", "Neutral", or "Bad" in terms of + relevance to the text exercise at hand. + Returns a tuple of lists of sentiments in each category. + """ + extract_sentiments_prompt = fmt_extract_sentiments_prompt( exercise_name=dto.exercise.title, course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, + previous_message=( + dto.conversation[-2].contents[0].text_content + if len(dto.conversation) > 1 + else None + ), user_input=dto.conversation[-1].contents[0].text_content, ) - guard_prompt = PyrisMessage( + extract_sentiments_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": guard_prompt}], + contents=[{"text_content": extract_sentiments_prompt}], + ) + response = self.request_handler.chat( + [extract_sentiments_prompt], CompletionArguments() ) - response = self.request_handler.chat([guard_prompt], CompletionArguments()) response = response.contents[0].text_content - return "yes" in response.lower() + print(f"Sentiments response:\n{response}") + sentiments = ([], [], []) + for line in response.split("\n"): + line = line.strip() + if line.startswith("Ok: "): + sentiments[0].append(line[4:]) + elif line.startswith("Neutral: "): + sentiments[1].append(line[10:]) + elif line.startswith("Bad: "): + sentiments[2].append(line[5:]) + return sentiments - def respond(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: - system_prompt = fmt_system_prompt( - exercise_name=dto.exercise.title, - course_name=dto.exercise.course.name, - course_description=dto.exercise.course.description, - problem_statement=dto.exercise.problem_statement, - start_date=str(dto.exercise.start_date), - end_date=str(dto.exercise.end_date), - current_date=str(datetime.now()), - current_submission=dto.current_submission, - ) + def respond( + self, + dto: TextExerciseChatPipelineExecutionDTO, + sentiments: (list[str], list[str], list[str]), + ) -> str: + """ + Actually respond to the user's input. + This takes the user's input and the conversation so far and generates a response. + """ system_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": system_prompt}], + contents=[ + { + "text_content": fmt_system_prompt( + exercise_name=dto.exercise.title, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + start_date=str(dto.exercise.start_date), + end_date=str(dto.exercise.end_date), + current_date=str(datetime.now()), + current_submission=dto.current_submission, + ) + } + ], ) - prompts = [system_prompt] + dto.conversation - - response = self.request_handler.chat( - prompts, CompletionArguments(temperature=0.4) - ) - return response.contents[0].text_content - - def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: - rejection_prompt = fmt_rejection_prompt( - exercise_name=dto.exercise.title, - course_name=dto.exercise.course.name, - course_description=dto.exercise.course.description, - problem_statement=dto.exercise.problem_statement, - user_input=dto.conversation[-1].contents[0].text_content, - ) - rejection_prompt = PyrisMessage( + sentiment_analysis = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": rejection_prompt}], + contents=[ + { + "text_content": fmt_sentiment_analysis_prompt( + respond_to=sentiments[0] + sentiments[1], + ignore=sentiments[2], + ) + } + ], ) + prompts = ( + [system_prompt] + + dto.conversation[:-1] + + [sentiment_analysis] + + dto.conversation[-1:] + ) + response = self.request_handler.chat( - [rejection_prompt], CompletionArguments(temperature=0.4) + prompts, CompletionArguments(temperature=0.4) ) return response.contents[0].text_content diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index da0078b8..30a73f13 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -233,7 +233,7 @@ def __init__( stage = len(stages) stages += [ StageDTO( - weight=20, + weight=30, state=StageStateEnum.NOT_STARTED, name="Thinking", ), From ba5e5343615adcd3e3b32a1a7ed545c1dd04db1e Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Thu, 10 Oct 2024 16:45:09 +0200 Subject: [PATCH 09/20] Format --- .../prompts/text_exercise_chat_prompts.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index d471ea8f..229e97eb 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -9,7 +9,7 @@ def fmt_extract_sentiments_prompt( return """ You extract and categorize sentiments of the user's input into three categories describing relevance and appropriateness in the context of a particular writing exercise. - + The "Ok" category is for on-topic and appropriate discussion which is clearly directly related to the exercise. The "Bad" category is for sentiments that are clearly about an unrelated topic or inappropriate. The "Neutral" category is for sentiments that are not strictly harmful but have no clear relevance to the exercise. @@ -18,21 +18,24 @@ def fmt_extract_sentiments_prompt( each separated by a newline. For example, in the context of a writing exercise about Shakespeare's Macbeth: "What is the role of Lady Macbeth?" -> "Ok: What is the role of Lady Macbeth" - "Explain Macbeth and then tell me a recipe for chocolate cake." -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake" - "Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!" + "Explain Macbeth and then tell me a recipe for chocolate cake." + -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake" + "Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" + -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!" "Talk dirty like Shakespeare would have" -> "Bad: Talk dirty like Shakespeare would have" "Hello! How are you?" -> "Neutral: Hello! How are you?" "How do I write a good essay?" -> "Ok: How do I write a good essay?" "What is the population of Serbia?" -> "Bad: What is the population of Serbia?" "Who won the 2020 Super Bowl? " -> "Bad: Who won the 2020 Super Bowl?" - "Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." + "Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." + -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." "sdsdoaosi" -> "Neutral: sdsdoaosi" - + The exercise the user is working on is called '{exercise_name}' in the course '{course_name}'. The course has the following description: {course_description} - + The writing exercise has the following problem statement: {problem_statement} @@ -82,7 +85,7 @@ def fmt_system_prompt( You point out specific issues in the student's writing and suggest improvements. You never provide answers or write the student's work for them. You are supportive, encouraging, and constructive in your feedback. - + The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. The course has the following description: {course_description} @@ -94,7 +97,7 @@ def fmt_system_prompt( This is the student's latest submission. (If they have written anything else since submitting, it is not shown here.) - + {current_submission} """.format( exercise_name=exercise_name, From bae6252d368fcbe2878a45f43a5c51ae6d9937ca Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Tue, 3 Sep 2024 17:21:21 +0200 Subject: [PATCH 10/20] Initial commit --- app/domain/data/text_exercise_dto.py | 15 +++++ ...xt_exercise_chat_pipeline_execution_dto.py | 10 ++++ .../prompts/text_exercise_chat_prompts.py | 35 +++++++++++ app/pipeline/text_exercise_chat_pipeline.py | 59 +++++++++++++++++++ app/web/routers/pipelines.py | 43 ++++++++++++++ app/web/status/status_update.py | 22 +++++++ 6 files changed, 184 insertions(+) create mode 100644 app/domain/data/text_exercise_dto.py create mode 100644 app/domain/text_exercise_chat_pipeline_execution_dto.py create mode 100644 app/pipeline/prompts/text_exercise_chat_prompts.py create mode 100644 app/pipeline/text_exercise_chat_pipeline.py diff --git a/app/domain/data/text_exercise_dto.py b/app/domain/data/text_exercise_dto.py new file mode 100644 index 00000000..7040b181 --- /dev/null +++ b/app/domain/data/text_exercise_dto.py @@ -0,0 +1,15 @@ +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel, Field + +from domain.data.course_dto import CourseDTO + + +class TextExerciseDTO(BaseModel): + id: int + name: str + course: CourseDTO + problem_statement: str = Field(alias="problemStatement") + start_date: Optional[datetime] = Field(alias="startDate", default=None) + end_date: Optional[datetime] = Field(alias="endDate", default=None) diff --git a/app/domain/text_exercise_chat_pipeline_execution_dto.py b/app/domain/text_exercise_chat_pipeline_execution_dto.py new file mode 100644 index 00000000..03ff7c19 --- /dev/null +++ b/app/domain/text_exercise_chat_pipeline_execution_dto.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel, Field + +from domain import PipelineExecutionDTO +from domain.data.text_exercise_dto import TextExerciseDTO + + +class TextExerciseChatPipelineExecutionDTO(BaseModel): + execution: PipelineExecutionDTO + exercise: TextExerciseDTO + current_answer: str = Field(alias="currentAnswer") diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py new file mode 100644 index 00000000..390cb954 --- /dev/null +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -0,0 +1,35 @@ +def system_prompt( + exercise_name: str, + course_name: str, + course_description: str, + problem_statement: str, + start_date: str, + end_date: str, + current_date: str, + current_answer: str, +) -> str: + return """ + The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. + The course has the following description: + {course_description} + + The exercise has the following problem statement: + {problem_statement} + + The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}. + + This is what the student has written so far: + {current_answer} + + You are a writing tutor. Provide feedback to the student on their response, + giving specific tips to better answer the problem statement. + """.format( + exercise_name=exercise_name, + course_name=course_name, + course_description=course_description, + problem_statement=problem_statement, + start_date=start_date, + end_date=end_date, + current_date=current_date, + current_answer=current_answer, + ) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py new file mode 100644 index 00000000..253504a2 --- /dev/null +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -0,0 +1,59 @@ +import logging +from datetime import datetime +from typing import Optional + +from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments +from app.pipeline import Pipeline +from domain import PyrisMessage, IrisMessageRole +from domain.data.text_message_content_dto import TextMessageContentDTO +from domain.text_exercise_chat_pipeline_execution_dto import ( + TextExerciseChatPipelineExecutionDTO, +) +from pipeline.prompts.text_exercise_chat_prompts import system_prompt +from web.status.status_update import TextExerciseChatCallback + +logger = logging.getLogger(__name__) + + +class TextExerciseChatPipeline(Pipeline): + callback: TextExerciseChatCallback + request_handler: CapabilityRequestHandler + + def __init__(self, callback: Optional[TextExerciseChatCallback] = None): + super().__init__(implementation_id="text_exercise_chat_pipeline_reference_impl") + self.callback = callback + self.request_handler = CapabilityRequestHandler( + requirements=RequirementList(context_length=8000) + ) + + def __call__( + self, + dto: TextExerciseChatPipelineExecutionDTO, + **kwargs, + ): + if not dto.exercise: + raise ValueError("Exercise is required") + + prompt = system_prompt( + exercise_name=dto.exercise.name, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + start_date=str(dto.exercise.start_date), + end_date=str(dto.exercise.end_date), + current_date=str(datetime.now()), + current_answer=dto.current_answer, + ) + prompt = PyrisMessage( + sender=IrisMessageRole.SYSTEM, + contents=[TextMessageContentDTO(text_content=prompt)], + ) + + # done building prompt + + response = self.request_handler.chat( + [prompt], CompletionArguments(temperature=0.4) + ) + response = response.contents[0].text_content + + self.callback.done(response) diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index eb198199..fae60926 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -21,6 +21,11 @@ from app.dependencies import TokenValidator from app.domain import FeatureDTO from app.pipeline.competency_extraction_pipeline import CompetencyExtractionPipeline +from domain.text_exercise_chat_pipeline_execution_dto import ( + TextExerciseChatPipelineExecutionDTO, +) +from pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline +from web.status.status_update import TextExerciseChatCallback router = APIRouter(prefix="/api/v1/pipelines", tags=["pipelines"]) logger = logging.getLogger(__name__) @@ -90,6 +95,44 @@ def run_course_chat_pipeline(variant: str, dto: CourseChatPipelineExecutionDTO): thread.start() +def run_text_exercise_chat_pipeline_worker(dto, variant): + try: + callback = TextExerciseChatCallback( + run_id=dto.settings.authentication_token, + base_url=dto.settings.artemis_base_url, + initial_stages=dto.initial_stages, + ) + match variant: + case "default" | "text_exercise_chat_pipeline_reference_impl": + pipeline = TextExerciseChatPipeline(callback=callback) + case _: + raise ValueError(f"Unknown variant: {variant}") + except Exception as e: + logger.error(f"Error preparing text exercise chat pipeline: {e}") + logger.error(traceback.format_exc()) + capture_exception(e) + return + + try: + pipeline(dto=dto) + except Exception as e: + logger.error(f"Error running text exercise chat pipeline: {e}") + logger.error(traceback.format_exc()) + callback.error("Fatal error.", exception=e) + + +@router.post( + "/text-exercise-chat/{variant}/run", + status_code=status.HTTP_202_ACCEPTED, + dependencies=[Depends(TokenValidator())], +) +def run_text_exercise_chat_pipeline( + variant: str, dto: TextExerciseChatPipelineExecutionDTO +): + thread = Thread(target=run_text_exercise_chat_pipeline_worker, args=(dto, variant)) + thread.start() + + def run_competency_extraction_pipeline_worker( dto: CompetencyExtractionPipelineExecutionDTO, _variant: str ): diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index 1f497f75..1ddf1ca9 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -218,6 +218,28 @@ def __init__( super().__init__(url, run_id, status, stage, current_stage_index) +class TextExerciseChatCallback(StatusCallback): + def __init__( + self, + run_id: str, + base_url: str, + initial_stages: List[StageDTO], + ): + url = f"{base_url}/api/public/pyris/pipelines/text-exercise-chat/runs/{run_id}/status" + stages = initial_stages or [] + stage = len(stages) + stages += [ + StageDTO( + weight=40, + state=StageStateEnum.NOT_STARTED, + name="Thinking", + ) + ] + super().__init__( + url, run_id, StatusUpdateDTO(stages=stages), stages[stage], stage + ) + + class CompetencyExtractionCallback(StatusCallback): def __init__( self, From 73407ed4cdd1300dedef513855137804078763a3 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Sun, 22 Sep 2024 19:19:12 +0200 Subject: [PATCH 11/20] Implement text exercise chat pipeline --- app/domain/data/text_exercise_dto.py | 2 +- .../text_exercise_chat_status_update_dto.py | 5 ++ ...xt_exercise_chat_pipeline_execution_dto.py | 5 +- .../prompts/text_exercise_chat_prompts.py | 60 ++++++++++++++-- app/pipeline/text_exercise_chat_pipeline.py | 68 +++++++++++++++---- app/web/routers/pipelines.py | 14 +++- app/web/status/status_update.py | 30 +++++--- 7 files changed, 153 insertions(+), 31 deletions(-) create mode 100644 app/domain/status/text_exercise_chat_status_update_dto.py diff --git a/app/domain/data/text_exercise_dto.py b/app/domain/data/text_exercise_dto.py index 7040b181..3fd15330 100644 --- a/app/domain/data/text_exercise_dto.py +++ b/app/domain/data/text_exercise_dto.py @@ -8,7 +8,7 @@ class TextExerciseDTO(BaseModel): id: int - name: str + title: str course: CourseDTO problem_statement: str = Field(alias="problemStatement") start_date: Optional[datetime] = Field(alias="startDate", default=None) diff --git a/app/domain/status/text_exercise_chat_status_update_dto.py b/app/domain/status/text_exercise_chat_status_update_dto.py new file mode 100644 index 00000000..dd063ff7 --- /dev/null +++ b/app/domain/status/text_exercise_chat_status_update_dto.py @@ -0,0 +1,5 @@ +from app.domain.status.status_update_dto import StatusUpdateDTO + + +class TextExerciseChatStatusUpdateDTO(StatusUpdateDTO): + result: str = [] diff --git a/app/domain/text_exercise_chat_pipeline_execution_dto.py b/app/domain/text_exercise_chat_pipeline_execution_dto.py index 03ff7c19..efae1adf 100644 --- a/app/domain/text_exercise_chat_pipeline_execution_dto.py +++ b/app/domain/text_exercise_chat_pipeline_execution_dto.py @@ -1,10 +1,11 @@ from pydantic import BaseModel, Field -from domain import PipelineExecutionDTO +from domain import PipelineExecutionDTO, PyrisMessage from domain.data.text_exercise_dto import TextExerciseDTO class TextExerciseChatPipelineExecutionDTO(BaseModel): execution: PipelineExecutionDTO exercise: TextExerciseDTO - current_answer: str = Field(alias="currentAnswer") + conversation: list[PyrisMessage] = Field(default=[]) + current_submission: str = Field(alias="currentSubmission", default="") diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index 390cb954..cf499241 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -1,4 +1,30 @@ -def system_prompt( +def fmt_guard_prompt( + exercise_name: str, + course_name: str, + course_description: str, + problem_statement: str, + user_input: str, +) -> str: + return """ + You check whether a user's input is on-topic and appropriate discourse in the context of a writing exercise. + The exercise is called '{exercise_name}' in the course '{course_name}'. + The course has the following description: + {course_description} + The exercise has the following problem statement: + {problem_statement} + The user says: + {user_input} + If this is on-topic and appropriate discussion, respond with "Yes". If not, respond with "No". + """.format( + exercise_name=exercise_name, + course_name=course_name, + course_description=course_description, + problem_statement=problem_statement, + user_input=user_input, + ) + + +def fmt_system_prompt( exercise_name: str, course_name: str, course_description: str, @@ -6,7 +32,7 @@ def system_prompt( start_date: str, end_date: str, current_date: str, - current_answer: str, + current_submission: str, ) -> str: return """ The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. @@ -19,7 +45,7 @@ def system_prompt( The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}. This is what the student has written so far: - {current_answer} + {current_submission} You are a writing tutor. Provide feedback to the student on their response, giving specific tips to better answer the problem statement. @@ -31,5 +57,31 @@ def system_prompt( start_date=start_date, end_date=end_date, current_date=current_date, - current_answer=current_answer, + current_submission=current_submission, + ) + + +def fmt_rejection_prompt( + exercise_name: str, + course_name: str, + course_description: str, + problem_statement: str, + user_input: str, +) -> str: + return """ + The user is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. + The course has the following description: + {course_description} + The exercise has the following problem statement: + {problem_statement} + The user has asked the following question: + {user_input} + The question is off-topic or inappropriate. + Briefly explain that you cannot help with their query, and prompt them to focus on the exercise. + """.format( + exercise_name=exercise_name, + course_name=course_name, + course_description=course_description, + problem_statement=problem_statement, + user_input=user_input, ) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 253504a2..38d5754d 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -5,11 +5,14 @@ from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments from app.pipeline import Pipeline from domain import PyrisMessage, IrisMessageRole -from domain.data.text_message_content_dto import TextMessageContentDTO from domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) -from pipeline.prompts.text_exercise_chat_prompts import system_prompt +from pipeline.prompts.text_exercise_chat_prompts import ( + fmt_system_prompt, + fmt_rejection_prompt, + fmt_guard_prompt, +) from web.status.status_update import TextExerciseChatCallback logger = logging.getLogger(__name__) @@ -34,26 +37,67 @@ def __call__( if not dto.exercise: raise ValueError("Exercise is required") - prompt = system_prompt( - exercise_name=dto.exercise.name, + should_respond = self.guard(dto) + self.callback.done("Responding" if should_respond else "Rejecting") + + if should_respond: + response = self.respond(dto) + else: + response = self.reject(dto) + + self.callback.done(final_result=response) + + def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool: + guard_prompt = fmt_guard_prompt( + exercise_name=dto.exercise.title, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + user_input=dto.current_submission, + ) + guard_prompt = PyrisMessage( + sender=IrisMessageRole.SYSTEM, + contents=[{"text_content": guard_prompt}], + ) + response = self.request_handler.chat([guard_prompt], CompletionArguments()) + response = response.contents[0].text_content + return "yes" in response.lower() + + def respond(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: + system_prompt = fmt_system_prompt( + exercise_name=dto.exercise.title, course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, start_date=str(dto.exercise.start_date), end_date=str(dto.exercise.end_date), current_date=str(datetime.now()), - current_answer=dto.current_answer, + current_submission=dto.current_submission, ) - prompt = PyrisMessage( + system_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[TextMessageContentDTO(text_content=prompt)], + contents=[{"text_content": system_prompt}], ) - - # done building prompt + prompts = [system_prompt] + dto.conversation response = self.request_handler.chat( - [prompt], CompletionArguments(temperature=0.4) + prompts, CompletionArguments(temperature=0.4) ) - response = response.contents[0].text_content + return response.contents[0].text_content - self.callback.done(response) + def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: + rejection_prompt = fmt_rejection_prompt( + exercise_name=dto.exercise.title, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + user_input=dto.current_submission, + ) + rejection_prompt = PyrisMessage( + sender=IrisMessageRole.SYSTEM, + contents=[{"text_content": rejection_prompt}], + ) + response = self.request_handler.chat( + [rejection_prompt], CompletionArguments(temperature=0.4) + ) + return response.contents[0].text_content diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index fae60926..c53ab668 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -98,9 +98,9 @@ def run_course_chat_pipeline(variant: str, dto: CourseChatPipelineExecutionDTO): def run_text_exercise_chat_pipeline_worker(dto, variant): try: callback = TextExerciseChatCallback( - run_id=dto.settings.authentication_token, - base_url=dto.settings.artemis_base_url, - initial_stages=dto.initial_stages, + run_id=dto.execution.settings.authentication_token, + base_url=dto.execution.settings.artemis_base_url, + initial_stages=dto.execution.initial_stages, ) match variant: case "default" | "text_exercise_chat_pipeline_reference_impl": @@ -193,6 +193,14 @@ def get_pipeline(feature: str): description="Default programming exercise chat variant.", ) ] + case "TEXT_EXERCISE_CHAT": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default text exercise chat variant.", + ) + ] case "COURSE_CHAT": return [ FeatureDTO( diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index 1ddf1ca9..da0078b8 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -5,18 +5,21 @@ import requests from abc import ABC -from ...domain.status.competency_extraction_status_update_dto import ( +from app.domain.status.competency_extraction_status_update_dto import ( CompetencyExtractionStatusUpdateDTO, ) -from ...domain.chat.course_chat.course_chat_status_update_dto import ( +from app.domain.chat.course_chat.course_chat_status_update_dto import ( CourseChatStatusUpdateDTO, ) -from ...domain.status.stage_state_dto import StageStateEnum -from ...domain.status.stage_dto import StageDTO -from ...domain.chat.exercise_chat.exercise_chat_status_update_dto import ( +from app.domain.status.stage_state_dto import StageStateEnum +from app.domain.status.stage_dto import StageDTO +from app.domain.status.text_exercise_chat_status_update_dto import ( + TextExerciseChatStatusUpdateDTO, +) +from app.domain.chat.exercise_chat.exercise_chat_status_update_dto import ( ExerciseChatStatusUpdateDTO, ) -from ...domain.status.status_update_dto import StatusUpdateDTO +from app.domain.status.status_update_dto import StatusUpdateDTO import logging logger = logging.getLogger(__name__) @@ -230,13 +233,22 @@ def __init__( stage = len(stages) stages += [ StageDTO( - weight=40, + weight=20, state=StageStateEnum.NOT_STARTED, name="Thinking", - ) + ), + StageDTO( + weight=20, + state=StageStateEnum.NOT_STARTED, + name="Responding", + ), ] super().__init__( - url, run_id, StatusUpdateDTO(stages=stages), stages[stage], stage + url, + run_id, + TextExerciseChatStatusUpdateDTO(stages=stages), + stages[stage], + stage, ) From 4d35c22a3177b43eeba5efdd30c340cb156651dd Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 18:33:50 +0200 Subject: [PATCH 12/20] Fix imports --- app/domain/data/text_exercise_dto.py | 2 +- app/domain/text_exercise_chat_pipeline_execution_dto.py | 4 ++-- app/pipeline/text_exercise_chat_pipeline.py | 4 ++-- app/web/routers/pipelines.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/domain/data/text_exercise_dto.py b/app/domain/data/text_exercise_dto.py index 3fd15330..098779f3 100644 --- a/app/domain/data/text_exercise_dto.py +++ b/app/domain/data/text_exercise_dto.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field -from domain.data.course_dto import CourseDTO +from app.domain.data.course_dto import CourseDTO class TextExerciseDTO(BaseModel): diff --git a/app/domain/text_exercise_chat_pipeline_execution_dto.py b/app/domain/text_exercise_chat_pipeline_execution_dto.py index efae1adf..65e8871c 100644 --- a/app/domain/text_exercise_chat_pipeline_execution_dto.py +++ b/app/domain/text_exercise_chat_pipeline_execution_dto.py @@ -1,7 +1,7 @@ from pydantic import BaseModel, Field -from domain import PipelineExecutionDTO, PyrisMessage -from domain.data.text_exercise_dto import TextExerciseDTO +from app.domain import PipelineExecutionDTO, PyrisMessage +from app.domain.data.text_exercise_dto import TextExerciseDTO class TextExerciseChatPipelineExecutionDTO(BaseModel): diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 38d5754d..5fc6cca6 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -4,8 +4,8 @@ from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments from app.pipeline import Pipeline -from domain import PyrisMessage, IrisMessageRole -from domain.text_exercise_chat_pipeline_execution_dto import ( +from app.domain import PyrisMessage, IrisMessageRole +from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) from pipeline.prompts.text_exercise_chat_prompts import ( diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index c53ab668..92171ed3 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -21,7 +21,7 @@ from app.dependencies import TokenValidator from app.domain import FeatureDTO from app.pipeline.competency_extraction_pipeline import CompetencyExtractionPipeline -from domain.text_exercise_chat_pipeline_execution_dto import ( +from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) from pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline From 2ecdcc7ae367cd9dd76dbb0cc175861a2cfc10eb Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 18:47:31 +0200 Subject: [PATCH 13/20] Fix imports --- app/pipeline/text_exercise_chat_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 5fc6cca6..3aad4625 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -8,12 +8,12 @@ from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) -from pipeline.prompts.text_exercise_chat_prompts import ( +from app.pipeline.prompts.text_exercise_chat_prompts import ( fmt_system_prompt, fmt_rejection_prompt, fmt_guard_prompt, ) -from web.status.status_update import TextExerciseChatCallback +from app.web.status.status_update import TextExerciseChatCallback logger = logging.getLogger(__name__) From b5884e010c64733e5247821d283b33c99789dcf6 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 18:54:49 +0200 Subject: [PATCH 14/20] Fix imports --- app/web/routers/pipelines.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index 92171ed3..7bc047c0 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -24,8 +24,8 @@ from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) -from pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline -from web.status.status_update import TextExerciseChatCallback +from app.pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline +from app.web.status.status_update import TextExerciseChatCallback router = APIRouter(prefix="/api/v1/pipelines", tags=["pipelines"]) logger = logging.getLogger(__name__) From 6805fa996765ddb5145cb08bba5b74ce1037189b Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 19:12:37 +0200 Subject: [PATCH 15/20] Fix --- app/pipeline/prompts/text_exercise_chat_prompts.py | 3 ++- app/pipeline/text_exercise_chat_pipeline.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index cf499241..2baf7590 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -14,7 +14,8 @@ def fmt_guard_prompt( {problem_statement} The user says: {user_input} - If this is on-topic and appropriate discussion, respond with "Yes". If not, respond with "No". + If this is on-topic and appropriate discussion, respond with "Yes". + If the user's input is clearly about something else or inappropriate, respond with "No". """.format( exercise_name=exercise_name, course_name=course_name, diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 3aad4625..9e2f2a59 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -53,7 +53,7 @@ def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool: course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, - user_input=dto.current_submission, + user_input=dto.conversation[-1].contents[0].text_content, ) guard_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, @@ -91,7 +91,7 @@ def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, - user_input=dto.current_submission, + user_input=dto.conversation[-1].contents[0].text_content, ) rejection_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, From c17278818e7a7a446cb509ea4be5db0444cb5c81 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Thu, 10 Oct 2024 13:33:20 +0200 Subject: [PATCH 16/20] Update pipeline --- .../prompts/text_exercise_chat_prompts.py | 94 ++++++++----- app/pipeline/text_exercise_chat_pipeline.py | 127 +++++++++++------- app/web/status/status_update.py | 2 +- 3 files changed, 140 insertions(+), 83 deletions(-) diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index 2baf7590..d471ea8f 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -1,30 +1,72 @@ -def fmt_guard_prompt( +def fmt_extract_sentiments_prompt( exercise_name: str, course_name: str, course_description: str, problem_statement: str, + previous_message: str, user_input: str, ) -> str: return """ - You check whether a user's input is on-topic and appropriate discourse in the context of a writing exercise. - The exercise is called '{exercise_name}' in the course '{course_name}'. + You extract and categorize sentiments of the user's input into three categories describing + relevance and appropriateness in the context of a particular writing exercise. + + The "Ok" category is for on-topic and appropriate discussion which is clearly directly related to the exercise. + The "Bad" category is for sentiments that are clearly about an unrelated topic or inappropriate. + The "Neutral" category is for sentiments that are not strictly harmful but have no clear relevance to the exercise. + + Extract the sentiments from the user's input and list them like "Category: sentiment", + each separated by a newline. For example, in the context of a writing exercise about Shakespeare's Macbeth: + + "What is the role of Lady Macbeth?" -> "Ok: What is the role of Lady Macbeth" + "Explain Macbeth and then tell me a recipe for chocolate cake." -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake" + "Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!" + "Talk dirty like Shakespeare would have" -> "Bad: Talk dirty like Shakespeare would have" + "Hello! How are you?" -> "Neutral: Hello! How are you?" + "How do I write a good essay?" -> "Ok: How do I write a good essay?" + "What is the population of Serbia?" -> "Bad: What is the population of Serbia?" + "Who won the 2020 Super Bowl? " -> "Bad: Who won the 2020 Super Bowl?" + "Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." + "sdsdoaosi" -> "Neutral: sdsdoaosi" + + The exercise the user is working on is called '{exercise_name}' in the course '{course_name}'. + The course has the following description: {course_description} - The exercise has the following problem statement: + + The writing exercise has the following problem statement: {problem_statement} - The user says: + + The previous thing said in the conversation was: + {previous_message} + + Given this context, what are the sentiments of the user's input? {user_input} - If this is on-topic and appropriate discussion, respond with "Yes". - If the user's input is clearly about something else or inappropriate, respond with "No". """.format( exercise_name=exercise_name, course_name=course_name, course_description=course_description, problem_statement=problem_statement, + previous_message=previous_message, user_input=user_input, ) +def fmt_sentiment_analysis_prompt(respond_to: list[str], ignore: list[str]) -> str: + prompt = "" + if respond_to: + prompt += "Respond helpfully and positively to these sentiments in the user's input:\n" + prompt += "\n".join(respond_to) + "\n\n" + if ignore: + prompt += """ + The following sentiments in the user's input are not relevant or appropriate to the writing exercise + and should be ignored. + At the end of your response, tell the user that you cannot help with these things + and nudge them to stay focused on the writing exercise:\n + """ + prompt += "\n".join(ignore) + return prompt + + def fmt_system_prompt( exercise_name: str, course_name: str, @@ -36,6 +78,11 @@ def fmt_system_prompt( current_submission: str, ) -> str: return """ + You are a writing tutor. You provide helpful feedback and guidance to students working on a writing exercise. + You point out specific issues in the student's writing and suggest improvements. + You never provide answers or write the student's work for them. + You are supportive, encouraging, and constructive in your feedback. + The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. The course has the following description: {course_description} @@ -45,11 +92,10 @@ def fmt_system_prompt( The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}. - This is what the student has written so far: + This is the student's latest submission. + (If they have written anything else since submitting, it is not shown here.) + {current_submission} - - You are a writing tutor. Provide feedback to the student on their response, - giving specific tips to better answer the problem statement. """.format( exercise_name=exercise_name, course_name=course_name, @@ -60,29 +106,3 @@ def fmt_system_prompt( current_date=current_date, current_submission=current_submission, ) - - -def fmt_rejection_prompt( - exercise_name: str, - course_name: str, - course_description: str, - problem_statement: str, - user_input: str, -) -> str: - return """ - The user is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. - The course has the following description: - {course_description} - The exercise has the following problem statement: - {problem_statement} - The user has asked the following question: - {user_input} - The question is off-topic or inappropriate. - Briefly explain that you cannot help with their query, and prompt them to focus on the exercise. - """.format( - exercise_name=exercise_name, - course_name=course_name, - course_description=course_description, - problem_statement=problem_statement, - user_input=user_input, - ) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 9e2f2a59..287f888d 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -10,10 +10,10 @@ ) from app.pipeline.prompts.text_exercise_chat_prompts import ( fmt_system_prompt, - fmt_rejection_prompt, - fmt_guard_prompt, + fmt_extract_sentiments_prompt, ) from app.web.status.status_update import TextExerciseChatCallback +from pipeline.prompts.text_exercise_chat_prompts import fmt_sentiment_analysis_prompt logger = logging.getLogger(__name__) @@ -34,70 +34,107 @@ def __call__( dto: TextExerciseChatPipelineExecutionDTO, **kwargs, ): + """ + Run the text exercise chat pipeline. + This consists of a sentiment analysis step followed by a response generation step. + """ if not dto.exercise: raise ValueError("Exercise is required") + if not dto.conversation: + raise ValueError("Conversation with at least one message is required") - should_respond = self.guard(dto) - self.callback.done("Responding" if should_respond else "Rejecting") - - if should_respond: - response = self.respond(dto) - else: - response = self.reject(dto) + sentiments = self.categorize_sentiments_by_relevance(dto) + print(f"Sentiments: {sentiments}") + self.callback.done("Responding") + response = self.respond(dto, sentiments) self.callback.done(final_result=response) - def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool: - guard_prompt = fmt_guard_prompt( + def categorize_sentiments_by_relevance( + self, dto: TextExerciseChatPipelineExecutionDTO + ) -> (list[str], list[str], list[str]): + """ + Extracts the sentiments from the user's input and categorizes them as "Ok", "Neutral", or "Bad" in terms of + relevance to the text exercise at hand. + Returns a tuple of lists of sentiments in each category. + """ + extract_sentiments_prompt = fmt_extract_sentiments_prompt( exercise_name=dto.exercise.title, course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, + previous_message=( + dto.conversation[-2].contents[0].text_content + if len(dto.conversation) > 1 + else None + ), user_input=dto.conversation[-1].contents[0].text_content, ) - guard_prompt = PyrisMessage( + extract_sentiments_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": guard_prompt}], + contents=[{"text_content": extract_sentiments_prompt}], + ) + response = self.request_handler.chat( + [extract_sentiments_prompt], CompletionArguments() ) - response = self.request_handler.chat([guard_prompt], CompletionArguments()) response = response.contents[0].text_content - return "yes" in response.lower() + print(f"Sentiments response:\n{response}") + sentiments = ([], [], []) + for line in response.split("\n"): + line = line.strip() + if line.startswith("Ok: "): + sentiments[0].append(line[4:]) + elif line.startswith("Neutral: "): + sentiments[1].append(line[10:]) + elif line.startswith("Bad: "): + sentiments[2].append(line[5:]) + return sentiments - def respond(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: - system_prompt = fmt_system_prompt( - exercise_name=dto.exercise.title, - course_name=dto.exercise.course.name, - course_description=dto.exercise.course.description, - problem_statement=dto.exercise.problem_statement, - start_date=str(dto.exercise.start_date), - end_date=str(dto.exercise.end_date), - current_date=str(datetime.now()), - current_submission=dto.current_submission, - ) + def respond( + self, + dto: TextExerciseChatPipelineExecutionDTO, + sentiments: (list[str], list[str], list[str]), + ) -> str: + """ + Actually respond to the user's input. + This takes the user's input and the conversation so far and generates a response. + """ system_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": system_prompt}], + contents=[ + { + "text_content": fmt_system_prompt( + exercise_name=dto.exercise.title, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + start_date=str(dto.exercise.start_date), + end_date=str(dto.exercise.end_date), + current_date=str(datetime.now()), + current_submission=dto.current_submission, + ) + } + ], ) - prompts = [system_prompt] + dto.conversation - - response = self.request_handler.chat( - prompts, CompletionArguments(temperature=0.4) - ) - return response.contents[0].text_content - - def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: - rejection_prompt = fmt_rejection_prompt( - exercise_name=dto.exercise.title, - course_name=dto.exercise.course.name, - course_description=dto.exercise.course.description, - problem_statement=dto.exercise.problem_statement, - user_input=dto.conversation[-1].contents[0].text_content, - ) - rejection_prompt = PyrisMessage( + sentiment_analysis = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": rejection_prompt}], + contents=[ + { + "text_content": fmt_sentiment_analysis_prompt( + respond_to=sentiments[0] + sentiments[1], + ignore=sentiments[2], + ) + } + ], ) + prompts = ( + [system_prompt] + + dto.conversation[:-1] + + [sentiment_analysis] + + dto.conversation[-1:] + ) + response = self.request_handler.chat( - [rejection_prompt], CompletionArguments(temperature=0.4) + prompts, CompletionArguments(temperature=0.4) ) return response.contents[0].text_content diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index da0078b8..30a73f13 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -233,7 +233,7 @@ def __init__( stage = len(stages) stages += [ StageDTO( - weight=20, + weight=30, state=StageStateEnum.NOT_STARTED, name="Thinking", ), From f1d2a226b7f586f7c44fec45f3c1179f8fc4637f Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Thu, 10 Oct 2024 16:45:09 +0200 Subject: [PATCH 17/20] Format --- .../prompts/text_exercise_chat_prompts.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index d471ea8f..229e97eb 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -9,7 +9,7 @@ def fmt_extract_sentiments_prompt( return """ You extract and categorize sentiments of the user's input into three categories describing relevance and appropriateness in the context of a particular writing exercise. - + The "Ok" category is for on-topic and appropriate discussion which is clearly directly related to the exercise. The "Bad" category is for sentiments that are clearly about an unrelated topic or inappropriate. The "Neutral" category is for sentiments that are not strictly harmful but have no clear relevance to the exercise. @@ -18,21 +18,24 @@ def fmt_extract_sentiments_prompt( each separated by a newline. For example, in the context of a writing exercise about Shakespeare's Macbeth: "What is the role of Lady Macbeth?" -> "Ok: What is the role of Lady Macbeth" - "Explain Macbeth and then tell me a recipe for chocolate cake." -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake" - "Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!" + "Explain Macbeth and then tell me a recipe for chocolate cake." + -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake" + "Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" + -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!" "Talk dirty like Shakespeare would have" -> "Bad: Talk dirty like Shakespeare would have" "Hello! How are you?" -> "Neutral: Hello! How are you?" "How do I write a good essay?" -> "Ok: How do I write a good essay?" "What is the population of Serbia?" -> "Bad: What is the population of Serbia?" "Who won the 2020 Super Bowl? " -> "Bad: Who won the 2020 Super Bowl?" - "Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." + "Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." + -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." "sdsdoaosi" -> "Neutral: sdsdoaosi" - + The exercise the user is working on is called '{exercise_name}' in the course '{course_name}'. The course has the following description: {course_description} - + The writing exercise has the following problem statement: {problem_statement} @@ -82,7 +85,7 @@ def fmt_system_prompt( You point out specific issues in the student's writing and suggest improvements. You never provide answers or write the student's work for them. You are supportive, encouraging, and constructive in your feedback. - + The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. The course has the following description: {course_description} @@ -94,7 +97,7 @@ def fmt_system_prompt( This is the student's latest submission. (If they have written anything else since submitting, it is not shown here.) - + {current_submission} """.format( exercise_name=exercise_name, From 28b02826a54def794afb77291628f7efafba8228 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Fri, 11 Oct 2024 17:04:04 +0200 Subject: [PATCH 18/20] Remove debug print statements --- app/pipeline/text_exercise_chat_pipeline.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 287f888d..9549f2eb 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -44,7 +44,6 @@ def __call__( raise ValueError("Conversation with at least one message is required") sentiments = self.categorize_sentiments_by_relevance(dto) - print(f"Sentiments: {sentiments}") self.callback.done("Responding") response = self.respond(dto, sentiments) @@ -78,7 +77,6 @@ def categorize_sentiments_by_relevance( [extract_sentiments_prompt], CompletionArguments() ) response = response.contents[0].text_content - print(f"Sentiments response:\n{response}") sentiments = ([], [], []) for line in response.split("\n"): line = line.strip() From 6ecedbaaaaaa6d093c470eb8df4449ba5037a7cd Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Fri, 11 Oct 2024 17:16:12 +0200 Subject: [PATCH 19/20] Apply coderabbit suggestions --- .../text_exercise_chat_status_update_dto.py | 2 +- .../prompts/text_exercise_chat_prompts.py | 19 ++++++++++++++----- app/pipeline/text_exercise_chat_pipeline.py | 6 +++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/app/domain/status/text_exercise_chat_status_update_dto.py b/app/domain/status/text_exercise_chat_status_update_dto.py index dd063ff7..a825e92f 100644 --- a/app/domain/status/text_exercise_chat_status_update_dto.py +++ b/app/domain/status/text_exercise_chat_status_update_dto.py @@ -2,4 +2,4 @@ class TextExerciseChatStatusUpdateDTO(StatusUpdateDTO): - result: str = [] + result: str diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index 229e97eb..477e4c4e 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -1,3 +1,6 @@ +import textwrap + + def fmt_extract_sentiments_prompt( exercise_name: str, course_name: str, @@ -6,7 +9,8 @@ def fmt_extract_sentiments_prompt( previous_message: str, user_input: str, ) -> str: - return """ + return textwrap.dedent( + """ You extract and categorize sentiments of the user's input into three categories describing relevance and appropriateness in the context of a particular writing exercise. @@ -44,7 +48,8 @@ def fmt_extract_sentiments_prompt( Given this context, what are the sentiments of the user's input? {user_input} - """.format( + """ + ).format( exercise_name=exercise_name, course_name=course_name, course_description=course_description, @@ -60,12 +65,14 @@ def fmt_sentiment_analysis_prompt(respond_to: list[str], ignore: list[str]) -> s prompt += "Respond helpfully and positively to these sentiments in the user's input:\n" prompt += "\n".join(respond_to) + "\n\n" if ignore: - prompt += """ + prompt += textwrap.dedent( + """ The following sentiments in the user's input are not relevant or appropriate to the writing exercise and should be ignored. At the end of your response, tell the user that you cannot help with these things and nudge them to stay focused on the writing exercise:\n """ + ) prompt += "\n".join(ignore) return prompt @@ -80,7 +87,8 @@ def fmt_system_prompt( current_date: str, current_submission: str, ) -> str: - return """ + return textwrap.dedent( + """ You are a writing tutor. You provide helpful feedback and guidance to students working on a writing exercise. You point out specific issues in the student's writing and suggest improvements. You never provide answers or write the student's work for them. @@ -99,7 +107,8 @@ def fmt_system_prompt( (If they have written anything else since submitting, it is not shown here.) {current_submission} - """.format( + """ + ).format( exercise_name=exercise_name, course_name=course_name, course_description=course_description, diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 9549f2eb..95e1ea1a 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -1,6 +1,6 @@ import logging from datetime import datetime -from typing import Optional +from typing import Optional, List, Tuple from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments from app.pipeline import Pipeline @@ -51,7 +51,7 @@ def __call__( def categorize_sentiments_by_relevance( self, dto: TextExerciseChatPipelineExecutionDTO - ) -> (list[str], list[str], list[str]): + ) -> Tuple[List[str], List[str], List[str]]: """ Extracts the sentiments from the user's input and categorizes them as "Ok", "Neutral", or "Bad" in terms of relevance to the text exercise at hand. @@ -91,7 +91,7 @@ def categorize_sentiments_by_relevance( def respond( self, dto: TextExerciseChatPipelineExecutionDTO, - sentiments: (list[str], list[str], list[str]), + sentiments: Tuple[List[str], List[str], List[str]], ) -> str: """ Actually respond to the user's input. From ecf7694f9c89b47dbabed50478616fc8719d3d19 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Fri, 11 Oct 2024 17:17:26 +0200 Subject: [PATCH 20/20] Update import --- app/pipeline/text_exercise_chat_pipeline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 95e1ea1a..5d27fc71 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -13,7 +13,9 @@ fmt_extract_sentiments_prompt, ) from app.web.status.status_update import TextExerciseChatCallback -from pipeline.prompts.text_exercise_chat_prompts import fmt_sentiment_analysis_prompt +from app.pipeline.prompts.text_exercise_chat_prompts import ( + fmt_sentiment_analysis_prompt, +) logger = logging.getLogger(__name__)