From 4e5e5387f45ed0cfb9a168b40aeb30d802b45db2 Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Sun, 25 Aug 2024 23:58:56 +0200 Subject: [PATCH 1/9] Support settings V3 --- app/domain/__init__.py | 1 + app/domain/feature_dto.py | 7 ++++++ app/web/routers/pipelines.py | 49 +++++++++++++++++++++++++++++++++--- 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 app/domain/feature_dto.py diff --git a/app/domain/__init__.py b/app/domain/__init__.py index 2f56f3f3..837b0741 100644 --- a/app/domain/__init__.py +++ b/app/domain/__init__.py @@ -11,3 +11,4 @@ ) from .pyris_message import PyrisMessage, IrisMessageRole from app.domain.data import image_message_content_dto +from app.domain.feature_dto import FeatureDTO diff --git a/app/domain/feature_dto.py b/app/domain/feature_dto.py new file mode 100644 index 00000000..997e2722 --- /dev/null +++ b/app/domain/feature_dto.py @@ -0,0 +1,7 @@ +from pydantic import BaseModel + + +class FeatureDTO(BaseModel): + id: str + name: str + description: str diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index 7ac9d3da..d6cae112 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -17,6 +17,7 @@ from app.pipeline.chat.course_chat_pipeline import CourseChatPipeline from app.pipeline.chat.exercise_chat_pipeline import ExerciseChatPipeline from app.dependencies import TokenValidator +from app.domain import FeatureDTO router = APIRouter(prefix="/api/v1/pipelines", tags=["pipelines"]) logger = logging.getLogger(__name__) @@ -86,9 +87,51 @@ def run_course_chat_pipeline(variant: str, dto: CourseChatPipelineExecutionDTO): thread.start() -@router.get("/{feature}") +@router.get("/{feature}/variants") def get_pipeline(feature: str): """ - Get the pipeline for the given feature. + Get the pipeline variants for the given feature. """ - return Response(status_code=status.HTTP_501_NOT_IMPLEMENTED) + match feature: + case "CHAT": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default chat variant.", + ) + ] + case "PROGRAMMING_EXERCISE_CHAT": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default programming exercise chat variant.", + ) + ] + case "COURSE_CHAT": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default course chat variant.", + ) + ] + case "COMPETENCY_GENERATION": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default competency generation variant.", + ) + ] + case "LECTURE_INGESTION": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default lecture ingestion variant.", + ) + ] + case _: + return Response(status_code=status.HTTP_400_BAD_REQUEST) From 6c79346b79971b9b82cbb2ea642ed5c250a54b90 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Tue, 3 Sep 2024 17:21:21 +0200 Subject: [PATCH 2/9] Initial commit --- app/domain/data/text_exercise_dto.py | 15 +++++ ...xt_exercise_chat_pipeline_execution_dto.py | 10 ++++ .../prompts/text_exercise_chat_prompts.py | 35 +++++++++++ app/pipeline/text_exercise_chat_pipeline.py | 59 +++++++++++++++++++ app/web/routers/pipelines.py | 43 ++++++++++++++ app/web/status/status_update.py | 22 +++++++ 6 files changed, 184 insertions(+) create mode 100644 app/domain/data/text_exercise_dto.py create mode 100644 app/domain/text_exercise_chat_pipeline_execution_dto.py create mode 100644 app/pipeline/prompts/text_exercise_chat_prompts.py create mode 100644 app/pipeline/text_exercise_chat_pipeline.py diff --git a/app/domain/data/text_exercise_dto.py b/app/domain/data/text_exercise_dto.py new file mode 100644 index 00000000..7040b181 --- /dev/null +++ b/app/domain/data/text_exercise_dto.py @@ -0,0 +1,15 @@ +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel, Field + +from domain.data.course_dto import CourseDTO + + +class TextExerciseDTO(BaseModel): + id: int + name: str + course: CourseDTO + problem_statement: str = Field(alias="problemStatement") + start_date: Optional[datetime] = Field(alias="startDate", default=None) + end_date: Optional[datetime] = Field(alias="endDate", default=None) diff --git a/app/domain/text_exercise_chat_pipeline_execution_dto.py b/app/domain/text_exercise_chat_pipeline_execution_dto.py new file mode 100644 index 00000000..03ff7c19 --- /dev/null +++ b/app/domain/text_exercise_chat_pipeline_execution_dto.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel, Field + +from domain import PipelineExecutionDTO +from domain.data.text_exercise_dto import TextExerciseDTO + + +class TextExerciseChatPipelineExecutionDTO(BaseModel): + execution: PipelineExecutionDTO + exercise: TextExerciseDTO + current_answer: str = Field(alias="currentAnswer") diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py new file mode 100644 index 00000000..390cb954 --- /dev/null +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -0,0 +1,35 @@ +def system_prompt( + exercise_name: str, + course_name: str, + course_description: str, + problem_statement: str, + start_date: str, + end_date: str, + current_date: str, + current_answer: str, +) -> str: + return """ + The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. + The course has the following description: + {course_description} + + The exercise has the following problem statement: + {problem_statement} + + The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}. + + This is what the student has written so far: + {current_answer} + + You are a writing tutor. Provide feedback to the student on their response, + giving specific tips to better answer the problem statement. + """.format( + exercise_name=exercise_name, + course_name=course_name, + course_description=course_description, + problem_statement=problem_statement, + start_date=start_date, + end_date=end_date, + current_date=current_date, + current_answer=current_answer, + ) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py new file mode 100644 index 00000000..253504a2 --- /dev/null +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -0,0 +1,59 @@ +import logging +from datetime import datetime +from typing import Optional + +from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments +from app.pipeline import Pipeline +from domain import PyrisMessage, IrisMessageRole +from domain.data.text_message_content_dto import TextMessageContentDTO +from domain.text_exercise_chat_pipeline_execution_dto import ( + TextExerciseChatPipelineExecutionDTO, +) +from pipeline.prompts.text_exercise_chat_prompts import system_prompt +from web.status.status_update import TextExerciseChatCallback + +logger = logging.getLogger(__name__) + + +class TextExerciseChatPipeline(Pipeline): + callback: TextExerciseChatCallback + request_handler: CapabilityRequestHandler + + def __init__(self, callback: Optional[TextExerciseChatCallback] = None): + super().__init__(implementation_id="text_exercise_chat_pipeline_reference_impl") + self.callback = callback + self.request_handler = CapabilityRequestHandler( + requirements=RequirementList(context_length=8000) + ) + + def __call__( + self, + dto: TextExerciseChatPipelineExecutionDTO, + **kwargs, + ): + if not dto.exercise: + raise ValueError("Exercise is required") + + prompt = system_prompt( + exercise_name=dto.exercise.name, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + start_date=str(dto.exercise.start_date), + end_date=str(dto.exercise.end_date), + current_date=str(datetime.now()), + current_answer=dto.current_answer, + ) + prompt = PyrisMessage( + sender=IrisMessageRole.SYSTEM, + contents=[TextMessageContentDTO(text_content=prompt)], + ) + + # done building prompt + + response = self.request_handler.chat( + [prompt], CompletionArguments(temperature=0.4) + ) + response = response.contents[0].text_content + + self.callback.done(response) diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index eb198199..fae60926 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -21,6 +21,11 @@ from app.dependencies import TokenValidator from app.domain import FeatureDTO from app.pipeline.competency_extraction_pipeline import CompetencyExtractionPipeline +from domain.text_exercise_chat_pipeline_execution_dto import ( + TextExerciseChatPipelineExecutionDTO, +) +from pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline +from web.status.status_update import TextExerciseChatCallback router = APIRouter(prefix="/api/v1/pipelines", tags=["pipelines"]) logger = logging.getLogger(__name__) @@ -90,6 +95,44 @@ def run_course_chat_pipeline(variant: str, dto: CourseChatPipelineExecutionDTO): thread.start() +def run_text_exercise_chat_pipeline_worker(dto, variant): + try: + callback = TextExerciseChatCallback( + run_id=dto.settings.authentication_token, + base_url=dto.settings.artemis_base_url, + initial_stages=dto.initial_stages, + ) + match variant: + case "default" | "text_exercise_chat_pipeline_reference_impl": + pipeline = TextExerciseChatPipeline(callback=callback) + case _: + raise ValueError(f"Unknown variant: {variant}") + except Exception as e: + logger.error(f"Error preparing text exercise chat pipeline: {e}") + logger.error(traceback.format_exc()) + capture_exception(e) + return + + try: + pipeline(dto=dto) + except Exception as e: + logger.error(f"Error running text exercise chat pipeline: {e}") + logger.error(traceback.format_exc()) + callback.error("Fatal error.", exception=e) + + +@router.post( + "/text-exercise-chat/{variant}/run", + status_code=status.HTTP_202_ACCEPTED, + dependencies=[Depends(TokenValidator())], +) +def run_text_exercise_chat_pipeline( + variant: str, dto: TextExerciseChatPipelineExecutionDTO +): + thread = Thread(target=run_text_exercise_chat_pipeline_worker, args=(dto, variant)) + thread.start() + + def run_competency_extraction_pipeline_worker( dto: CompetencyExtractionPipelineExecutionDTO, _variant: str ): diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index 1f497f75..1ddf1ca9 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -218,6 +218,28 @@ def __init__( super().__init__(url, run_id, status, stage, current_stage_index) +class TextExerciseChatCallback(StatusCallback): + def __init__( + self, + run_id: str, + base_url: str, + initial_stages: List[StageDTO], + ): + url = f"{base_url}/api/public/pyris/pipelines/text-exercise-chat/runs/{run_id}/status" + stages = initial_stages or [] + stage = len(stages) + stages += [ + StageDTO( + weight=40, + state=StageStateEnum.NOT_STARTED, + name="Thinking", + ) + ] + super().__init__( + url, run_id, StatusUpdateDTO(stages=stages), stages[stage], stage + ) + + class CompetencyExtractionCallback(StatusCallback): def __init__( self, From 52d6bf8b27b9f76d03c14dd4c59ab412d39b94e8 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Sun, 22 Sep 2024 19:19:12 +0200 Subject: [PATCH 3/9] Implement text exercise chat pipeline --- app/domain/data/text_exercise_dto.py | 2 +- .../text_exercise_chat_status_update_dto.py | 5 ++ ...xt_exercise_chat_pipeline_execution_dto.py | 5 +- .../prompts/text_exercise_chat_prompts.py | 60 ++++++++++++++-- app/pipeline/text_exercise_chat_pipeline.py | 68 +++++++++++++++---- app/web/routers/pipelines.py | 14 +++- app/web/status/status_update.py | 30 +++++--- 7 files changed, 153 insertions(+), 31 deletions(-) create mode 100644 app/domain/status/text_exercise_chat_status_update_dto.py diff --git a/app/domain/data/text_exercise_dto.py b/app/domain/data/text_exercise_dto.py index 7040b181..3fd15330 100644 --- a/app/domain/data/text_exercise_dto.py +++ b/app/domain/data/text_exercise_dto.py @@ -8,7 +8,7 @@ class TextExerciseDTO(BaseModel): id: int - name: str + title: str course: CourseDTO problem_statement: str = Field(alias="problemStatement") start_date: Optional[datetime] = Field(alias="startDate", default=None) diff --git a/app/domain/status/text_exercise_chat_status_update_dto.py b/app/domain/status/text_exercise_chat_status_update_dto.py new file mode 100644 index 00000000..dd063ff7 --- /dev/null +++ b/app/domain/status/text_exercise_chat_status_update_dto.py @@ -0,0 +1,5 @@ +from app.domain.status.status_update_dto import StatusUpdateDTO + + +class TextExerciseChatStatusUpdateDTO(StatusUpdateDTO): + result: str = [] diff --git a/app/domain/text_exercise_chat_pipeline_execution_dto.py b/app/domain/text_exercise_chat_pipeline_execution_dto.py index 03ff7c19..efae1adf 100644 --- a/app/domain/text_exercise_chat_pipeline_execution_dto.py +++ b/app/domain/text_exercise_chat_pipeline_execution_dto.py @@ -1,10 +1,11 @@ from pydantic import BaseModel, Field -from domain import PipelineExecutionDTO +from domain import PipelineExecutionDTO, PyrisMessage from domain.data.text_exercise_dto import TextExerciseDTO class TextExerciseChatPipelineExecutionDTO(BaseModel): execution: PipelineExecutionDTO exercise: TextExerciseDTO - current_answer: str = Field(alias="currentAnswer") + conversation: list[PyrisMessage] = Field(default=[]) + current_submission: str = Field(alias="currentSubmission", default="") diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index 390cb954..cf499241 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -1,4 +1,30 @@ -def system_prompt( +def fmt_guard_prompt( + exercise_name: str, + course_name: str, + course_description: str, + problem_statement: str, + user_input: str, +) -> str: + return """ + You check whether a user's input is on-topic and appropriate discourse in the context of a writing exercise. + The exercise is called '{exercise_name}' in the course '{course_name}'. + The course has the following description: + {course_description} + The exercise has the following problem statement: + {problem_statement} + The user says: + {user_input} + If this is on-topic and appropriate discussion, respond with "Yes". If not, respond with "No". + """.format( + exercise_name=exercise_name, + course_name=course_name, + course_description=course_description, + problem_statement=problem_statement, + user_input=user_input, + ) + + +def fmt_system_prompt( exercise_name: str, course_name: str, course_description: str, @@ -6,7 +32,7 @@ def system_prompt( start_date: str, end_date: str, current_date: str, - current_answer: str, + current_submission: str, ) -> str: return """ The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. @@ -19,7 +45,7 @@ def system_prompt( The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}. This is what the student has written so far: - {current_answer} + {current_submission} You are a writing tutor. Provide feedback to the student on their response, giving specific tips to better answer the problem statement. @@ -31,5 +57,31 @@ def system_prompt( start_date=start_date, end_date=end_date, current_date=current_date, - current_answer=current_answer, + current_submission=current_submission, + ) + + +def fmt_rejection_prompt( + exercise_name: str, + course_name: str, + course_description: str, + problem_statement: str, + user_input: str, +) -> str: + return """ + The user is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. + The course has the following description: + {course_description} + The exercise has the following problem statement: + {problem_statement} + The user has asked the following question: + {user_input} + The question is off-topic or inappropriate. + Briefly explain that you cannot help with their query, and prompt them to focus on the exercise. + """.format( + exercise_name=exercise_name, + course_name=course_name, + course_description=course_description, + problem_statement=problem_statement, + user_input=user_input, ) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 253504a2..38d5754d 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -5,11 +5,14 @@ from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments from app.pipeline import Pipeline from domain import PyrisMessage, IrisMessageRole -from domain.data.text_message_content_dto import TextMessageContentDTO from domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) -from pipeline.prompts.text_exercise_chat_prompts import system_prompt +from pipeline.prompts.text_exercise_chat_prompts import ( + fmt_system_prompt, + fmt_rejection_prompt, + fmt_guard_prompt, +) from web.status.status_update import TextExerciseChatCallback logger = logging.getLogger(__name__) @@ -34,26 +37,67 @@ def __call__( if not dto.exercise: raise ValueError("Exercise is required") - prompt = system_prompt( - exercise_name=dto.exercise.name, + should_respond = self.guard(dto) + self.callback.done("Responding" if should_respond else "Rejecting") + + if should_respond: + response = self.respond(dto) + else: + response = self.reject(dto) + + self.callback.done(final_result=response) + + def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool: + guard_prompt = fmt_guard_prompt( + exercise_name=dto.exercise.title, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + user_input=dto.current_submission, + ) + guard_prompt = PyrisMessage( + sender=IrisMessageRole.SYSTEM, + contents=[{"text_content": guard_prompt}], + ) + response = self.request_handler.chat([guard_prompt], CompletionArguments()) + response = response.contents[0].text_content + return "yes" in response.lower() + + def respond(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: + system_prompt = fmt_system_prompt( + exercise_name=dto.exercise.title, course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, start_date=str(dto.exercise.start_date), end_date=str(dto.exercise.end_date), current_date=str(datetime.now()), - current_answer=dto.current_answer, + current_submission=dto.current_submission, ) - prompt = PyrisMessage( + system_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[TextMessageContentDTO(text_content=prompt)], + contents=[{"text_content": system_prompt}], ) - - # done building prompt + prompts = [system_prompt] + dto.conversation response = self.request_handler.chat( - [prompt], CompletionArguments(temperature=0.4) + prompts, CompletionArguments(temperature=0.4) ) - response = response.contents[0].text_content + return response.contents[0].text_content - self.callback.done(response) + def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: + rejection_prompt = fmt_rejection_prompt( + exercise_name=dto.exercise.title, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + user_input=dto.current_submission, + ) + rejection_prompt = PyrisMessage( + sender=IrisMessageRole.SYSTEM, + contents=[{"text_content": rejection_prompt}], + ) + response = self.request_handler.chat( + [rejection_prompt], CompletionArguments(temperature=0.4) + ) + return response.contents[0].text_content diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index fae60926..c53ab668 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -98,9 +98,9 @@ def run_course_chat_pipeline(variant: str, dto: CourseChatPipelineExecutionDTO): def run_text_exercise_chat_pipeline_worker(dto, variant): try: callback = TextExerciseChatCallback( - run_id=dto.settings.authentication_token, - base_url=dto.settings.artemis_base_url, - initial_stages=dto.initial_stages, + run_id=dto.execution.settings.authentication_token, + base_url=dto.execution.settings.artemis_base_url, + initial_stages=dto.execution.initial_stages, ) match variant: case "default" | "text_exercise_chat_pipeline_reference_impl": @@ -193,6 +193,14 @@ def get_pipeline(feature: str): description="Default programming exercise chat variant.", ) ] + case "TEXT_EXERCISE_CHAT": + return [ + FeatureDTO( + id="default", + name="Default Variant", + description="Default text exercise chat variant.", + ) + ] case "COURSE_CHAT": return [ FeatureDTO( diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index 1ddf1ca9..da0078b8 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -5,18 +5,21 @@ import requests from abc import ABC -from ...domain.status.competency_extraction_status_update_dto import ( +from app.domain.status.competency_extraction_status_update_dto import ( CompetencyExtractionStatusUpdateDTO, ) -from ...domain.chat.course_chat.course_chat_status_update_dto import ( +from app.domain.chat.course_chat.course_chat_status_update_dto import ( CourseChatStatusUpdateDTO, ) -from ...domain.status.stage_state_dto import StageStateEnum -from ...domain.status.stage_dto import StageDTO -from ...domain.chat.exercise_chat.exercise_chat_status_update_dto import ( +from app.domain.status.stage_state_dto import StageStateEnum +from app.domain.status.stage_dto import StageDTO +from app.domain.status.text_exercise_chat_status_update_dto import ( + TextExerciseChatStatusUpdateDTO, +) +from app.domain.chat.exercise_chat.exercise_chat_status_update_dto import ( ExerciseChatStatusUpdateDTO, ) -from ...domain.status.status_update_dto import StatusUpdateDTO +from app.domain.status.status_update_dto import StatusUpdateDTO import logging logger = logging.getLogger(__name__) @@ -230,13 +233,22 @@ def __init__( stage = len(stages) stages += [ StageDTO( - weight=40, + weight=20, state=StageStateEnum.NOT_STARTED, name="Thinking", - ) + ), + StageDTO( + weight=20, + state=StageStateEnum.NOT_STARTED, + name="Responding", + ), ] super().__init__( - url, run_id, StatusUpdateDTO(stages=stages), stages[stage], stage + url, + run_id, + TextExerciseChatStatusUpdateDTO(stages=stages), + stages[stage], + stage, ) From ff3f259c0652baebf2d94afba86512f31eb96d29 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 18:33:50 +0200 Subject: [PATCH 4/9] Fix imports --- app/domain/data/text_exercise_dto.py | 2 +- app/domain/text_exercise_chat_pipeline_execution_dto.py | 4 ++-- app/pipeline/text_exercise_chat_pipeline.py | 4 ++-- app/web/routers/pipelines.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/domain/data/text_exercise_dto.py b/app/domain/data/text_exercise_dto.py index 3fd15330..098779f3 100644 --- a/app/domain/data/text_exercise_dto.py +++ b/app/domain/data/text_exercise_dto.py @@ -3,7 +3,7 @@ from pydantic import BaseModel, Field -from domain.data.course_dto import CourseDTO +from app.domain.data.course_dto import CourseDTO class TextExerciseDTO(BaseModel): diff --git a/app/domain/text_exercise_chat_pipeline_execution_dto.py b/app/domain/text_exercise_chat_pipeline_execution_dto.py index efae1adf..65e8871c 100644 --- a/app/domain/text_exercise_chat_pipeline_execution_dto.py +++ b/app/domain/text_exercise_chat_pipeline_execution_dto.py @@ -1,7 +1,7 @@ from pydantic import BaseModel, Field -from domain import PipelineExecutionDTO, PyrisMessage -from domain.data.text_exercise_dto import TextExerciseDTO +from app.domain import PipelineExecutionDTO, PyrisMessage +from app.domain.data.text_exercise_dto import TextExerciseDTO class TextExerciseChatPipelineExecutionDTO(BaseModel): diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 38d5754d..5fc6cca6 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -4,8 +4,8 @@ from app.llm import CapabilityRequestHandler, RequirementList, CompletionArguments from app.pipeline import Pipeline -from domain import PyrisMessage, IrisMessageRole -from domain.text_exercise_chat_pipeline_execution_dto import ( +from app.domain import PyrisMessage, IrisMessageRole +from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) from pipeline.prompts.text_exercise_chat_prompts import ( diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index c53ab668..92171ed3 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -21,7 +21,7 @@ from app.dependencies import TokenValidator from app.domain import FeatureDTO from app.pipeline.competency_extraction_pipeline import CompetencyExtractionPipeline -from domain.text_exercise_chat_pipeline_execution_dto import ( +from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) from pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline From 72067852ba42933d9cbcae534d5dcc38d999d8d2 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 18:47:31 +0200 Subject: [PATCH 5/9] Fix imports --- app/pipeline/text_exercise_chat_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 5fc6cca6..3aad4625 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -8,12 +8,12 @@ from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) -from pipeline.prompts.text_exercise_chat_prompts import ( +from app.pipeline.prompts.text_exercise_chat_prompts import ( fmt_system_prompt, fmt_rejection_prompt, fmt_guard_prompt, ) -from web.status.status_update import TextExerciseChatCallback +from app.web.status.status_update import TextExerciseChatCallback logger = logging.getLogger(__name__) From 80a846cd5838d4b99cec58fec8ba5ceba88c873e Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 18:54:49 +0200 Subject: [PATCH 6/9] Fix imports --- app/web/routers/pipelines.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/web/routers/pipelines.py b/app/web/routers/pipelines.py index 92171ed3..7bc047c0 100644 --- a/app/web/routers/pipelines.py +++ b/app/web/routers/pipelines.py @@ -24,8 +24,8 @@ from app.domain.text_exercise_chat_pipeline_execution_dto import ( TextExerciseChatPipelineExecutionDTO, ) -from pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline -from web.status.status_update import TextExerciseChatCallback +from app.pipeline.text_exercise_chat_pipeline import TextExerciseChatPipeline +from app.web.status.status_update import TextExerciseChatCallback router = APIRouter(prefix="/api/v1/pipelines", tags=["pipelines"]) logger = logging.getLogger(__name__) From 4042ab52e99367ef70eb1527522596192493b512 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Wed, 9 Oct 2024 19:12:37 +0200 Subject: [PATCH 7/9] Fix --- app/pipeline/prompts/text_exercise_chat_prompts.py | 3 ++- app/pipeline/text_exercise_chat_pipeline.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index cf499241..2baf7590 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -14,7 +14,8 @@ def fmt_guard_prompt( {problem_statement} The user says: {user_input} - If this is on-topic and appropriate discussion, respond with "Yes". If not, respond with "No". + If this is on-topic and appropriate discussion, respond with "Yes". + If the user's input is clearly about something else or inappropriate, respond with "No". """.format( exercise_name=exercise_name, course_name=course_name, diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 3aad4625..9e2f2a59 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -53,7 +53,7 @@ def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool: course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, - user_input=dto.current_submission, + user_input=dto.conversation[-1].contents[0].text_content, ) guard_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, @@ -91,7 +91,7 @@ def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, - user_input=dto.current_submission, + user_input=dto.conversation[-1].contents[0].text_content, ) rejection_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, From ed4d5dd7ea750206ca80c7128ee6155f8f0279d5 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Thu, 10 Oct 2024 13:33:20 +0200 Subject: [PATCH 8/9] Update pipeline --- .../prompts/text_exercise_chat_prompts.py | 94 ++++++++----- app/pipeline/text_exercise_chat_pipeline.py | 127 +++++++++++------- app/web/status/status_update.py | 2 +- 3 files changed, 140 insertions(+), 83 deletions(-) diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index 2baf7590..d471ea8f 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -1,30 +1,72 @@ -def fmt_guard_prompt( +def fmt_extract_sentiments_prompt( exercise_name: str, course_name: str, course_description: str, problem_statement: str, + previous_message: str, user_input: str, ) -> str: return """ - You check whether a user's input is on-topic and appropriate discourse in the context of a writing exercise. - The exercise is called '{exercise_name}' in the course '{course_name}'. + You extract and categorize sentiments of the user's input into three categories describing + relevance and appropriateness in the context of a particular writing exercise. + + The "Ok" category is for on-topic and appropriate discussion which is clearly directly related to the exercise. + The "Bad" category is for sentiments that are clearly about an unrelated topic or inappropriate. + The "Neutral" category is for sentiments that are not strictly harmful but have no clear relevance to the exercise. + + Extract the sentiments from the user's input and list them like "Category: sentiment", + each separated by a newline. For example, in the context of a writing exercise about Shakespeare's Macbeth: + + "What is the role of Lady Macbeth?" -> "Ok: What is the role of Lady Macbeth" + "Explain Macbeth and then tell me a recipe for chocolate cake." -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake" + "Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!" + "Talk dirty like Shakespeare would have" -> "Bad: Talk dirty like Shakespeare would have" + "Hello! How are you?" -> "Neutral: Hello! How are you?" + "How do I write a good essay?" -> "Ok: How do I write a good essay?" + "What is the population of Serbia?" -> "Bad: What is the population of Serbia?" + "Who won the 2020 Super Bowl? " -> "Bad: Who won the 2020 Super Bowl?" + "Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." + "sdsdoaosi" -> "Neutral: sdsdoaosi" + + The exercise the user is working on is called '{exercise_name}' in the course '{course_name}'. + The course has the following description: {course_description} - The exercise has the following problem statement: + + The writing exercise has the following problem statement: {problem_statement} - The user says: + + The previous thing said in the conversation was: + {previous_message} + + Given this context, what are the sentiments of the user's input? {user_input} - If this is on-topic and appropriate discussion, respond with "Yes". - If the user's input is clearly about something else or inappropriate, respond with "No". """.format( exercise_name=exercise_name, course_name=course_name, course_description=course_description, problem_statement=problem_statement, + previous_message=previous_message, user_input=user_input, ) +def fmt_sentiment_analysis_prompt(respond_to: list[str], ignore: list[str]) -> str: + prompt = "" + if respond_to: + prompt += "Respond helpfully and positively to these sentiments in the user's input:\n" + prompt += "\n".join(respond_to) + "\n\n" + if ignore: + prompt += """ + The following sentiments in the user's input are not relevant or appropriate to the writing exercise + and should be ignored. + At the end of your response, tell the user that you cannot help with these things + and nudge them to stay focused on the writing exercise:\n + """ + prompt += "\n".join(ignore) + return prompt + + def fmt_system_prompt( exercise_name: str, course_name: str, @@ -36,6 +78,11 @@ def fmt_system_prompt( current_submission: str, ) -> str: return """ + You are a writing tutor. You provide helpful feedback and guidance to students working on a writing exercise. + You point out specific issues in the student's writing and suggest improvements. + You never provide answers or write the student's work for them. + You are supportive, encouraging, and constructive in your feedback. + The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. The course has the following description: {course_description} @@ -45,11 +92,10 @@ def fmt_system_prompt( The exercise began on {start_date} and will end on {end_date}. The current date is {current_date}. - This is what the student has written so far: + This is the student's latest submission. + (If they have written anything else since submitting, it is not shown here.) + {current_submission} - - You are a writing tutor. Provide feedback to the student on their response, - giving specific tips to better answer the problem statement. """.format( exercise_name=exercise_name, course_name=course_name, @@ -60,29 +106,3 @@ def fmt_system_prompt( current_date=current_date, current_submission=current_submission, ) - - -def fmt_rejection_prompt( - exercise_name: str, - course_name: str, - course_description: str, - problem_statement: str, - user_input: str, -) -> str: - return """ - The user is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. - The course has the following description: - {course_description} - The exercise has the following problem statement: - {problem_statement} - The user has asked the following question: - {user_input} - The question is off-topic or inappropriate. - Briefly explain that you cannot help with their query, and prompt them to focus on the exercise. - """.format( - exercise_name=exercise_name, - course_name=course_name, - course_description=course_description, - problem_statement=problem_statement, - user_input=user_input, - ) diff --git a/app/pipeline/text_exercise_chat_pipeline.py b/app/pipeline/text_exercise_chat_pipeline.py index 9e2f2a59..287f888d 100644 --- a/app/pipeline/text_exercise_chat_pipeline.py +++ b/app/pipeline/text_exercise_chat_pipeline.py @@ -10,10 +10,10 @@ ) from app.pipeline.prompts.text_exercise_chat_prompts import ( fmt_system_prompt, - fmt_rejection_prompt, - fmt_guard_prompt, + fmt_extract_sentiments_prompt, ) from app.web.status.status_update import TextExerciseChatCallback +from pipeline.prompts.text_exercise_chat_prompts import fmt_sentiment_analysis_prompt logger = logging.getLogger(__name__) @@ -34,70 +34,107 @@ def __call__( dto: TextExerciseChatPipelineExecutionDTO, **kwargs, ): + """ + Run the text exercise chat pipeline. + This consists of a sentiment analysis step followed by a response generation step. + """ if not dto.exercise: raise ValueError("Exercise is required") + if not dto.conversation: + raise ValueError("Conversation with at least one message is required") - should_respond = self.guard(dto) - self.callback.done("Responding" if should_respond else "Rejecting") - - if should_respond: - response = self.respond(dto) - else: - response = self.reject(dto) + sentiments = self.categorize_sentiments_by_relevance(dto) + print(f"Sentiments: {sentiments}") + self.callback.done("Responding") + response = self.respond(dto, sentiments) self.callback.done(final_result=response) - def guard(self, dto: TextExerciseChatPipelineExecutionDTO) -> bool: - guard_prompt = fmt_guard_prompt( + def categorize_sentiments_by_relevance( + self, dto: TextExerciseChatPipelineExecutionDTO + ) -> (list[str], list[str], list[str]): + """ + Extracts the sentiments from the user's input and categorizes them as "Ok", "Neutral", or "Bad" in terms of + relevance to the text exercise at hand. + Returns a tuple of lists of sentiments in each category. + """ + extract_sentiments_prompt = fmt_extract_sentiments_prompt( exercise_name=dto.exercise.title, course_name=dto.exercise.course.name, course_description=dto.exercise.course.description, problem_statement=dto.exercise.problem_statement, + previous_message=( + dto.conversation[-2].contents[0].text_content + if len(dto.conversation) > 1 + else None + ), user_input=dto.conversation[-1].contents[0].text_content, ) - guard_prompt = PyrisMessage( + extract_sentiments_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": guard_prompt}], + contents=[{"text_content": extract_sentiments_prompt}], + ) + response = self.request_handler.chat( + [extract_sentiments_prompt], CompletionArguments() ) - response = self.request_handler.chat([guard_prompt], CompletionArguments()) response = response.contents[0].text_content - return "yes" in response.lower() + print(f"Sentiments response:\n{response}") + sentiments = ([], [], []) + for line in response.split("\n"): + line = line.strip() + if line.startswith("Ok: "): + sentiments[0].append(line[4:]) + elif line.startswith("Neutral: "): + sentiments[1].append(line[10:]) + elif line.startswith("Bad: "): + sentiments[2].append(line[5:]) + return sentiments - def respond(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: - system_prompt = fmt_system_prompt( - exercise_name=dto.exercise.title, - course_name=dto.exercise.course.name, - course_description=dto.exercise.course.description, - problem_statement=dto.exercise.problem_statement, - start_date=str(dto.exercise.start_date), - end_date=str(dto.exercise.end_date), - current_date=str(datetime.now()), - current_submission=dto.current_submission, - ) + def respond( + self, + dto: TextExerciseChatPipelineExecutionDTO, + sentiments: (list[str], list[str], list[str]), + ) -> str: + """ + Actually respond to the user's input. + This takes the user's input and the conversation so far and generates a response. + """ system_prompt = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": system_prompt}], + contents=[ + { + "text_content": fmt_system_prompt( + exercise_name=dto.exercise.title, + course_name=dto.exercise.course.name, + course_description=dto.exercise.course.description, + problem_statement=dto.exercise.problem_statement, + start_date=str(dto.exercise.start_date), + end_date=str(dto.exercise.end_date), + current_date=str(datetime.now()), + current_submission=dto.current_submission, + ) + } + ], ) - prompts = [system_prompt] + dto.conversation - - response = self.request_handler.chat( - prompts, CompletionArguments(temperature=0.4) - ) - return response.contents[0].text_content - - def reject(self, dto: TextExerciseChatPipelineExecutionDTO) -> str: - rejection_prompt = fmt_rejection_prompt( - exercise_name=dto.exercise.title, - course_name=dto.exercise.course.name, - course_description=dto.exercise.course.description, - problem_statement=dto.exercise.problem_statement, - user_input=dto.conversation[-1].contents[0].text_content, - ) - rejection_prompt = PyrisMessage( + sentiment_analysis = PyrisMessage( sender=IrisMessageRole.SYSTEM, - contents=[{"text_content": rejection_prompt}], + contents=[ + { + "text_content": fmt_sentiment_analysis_prompt( + respond_to=sentiments[0] + sentiments[1], + ignore=sentiments[2], + ) + } + ], ) + prompts = ( + [system_prompt] + + dto.conversation[:-1] + + [sentiment_analysis] + + dto.conversation[-1:] + ) + response = self.request_handler.chat( - [rejection_prompt], CompletionArguments(temperature=0.4) + prompts, CompletionArguments(temperature=0.4) ) return response.contents[0].text_content diff --git a/app/web/status/status_update.py b/app/web/status/status_update.py index da0078b8..30a73f13 100644 --- a/app/web/status/status_update.py +++ b/app/web/status/status_update.py @@ -233,7 +233,7 @@ def __init__( stage = len(stages) stages += [ StageDTO( - weight=20, + weight=30, state=StageStateEnum.NOT_STARTED, name="Thinking", ), From ba5e5343615adcd3e3b32a1a7ed545c1dd04db1e Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Thu, 10 Oct 2024 16:45:09 +0200 Subject: [PATCH 9/9] Format --- .../prompts/text_exercise_chat_prompts.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/app/pipeline/prompts/text_exercise_chat_prompts.py b/app/pipeline/prompts/text_exercise_chat_prompts.py index d471ea8f..229e97eb 100644 --- a/app/pipeline/prompts/text_exercise_chat_prompts.py +++ b/app/pipeline/prompts/text_exercise_chat_prompts.py @@ -9,7 +9,7 @@ def fmt_extract_sentiments_prompt( return """ You extract and categorize sentiments of the user's input into three categories describing relevance and appropriateness in the context of a particular writing exercise. - + The "Ok" category is for on-topic and appropriate discussion which is clearly directly related to the exercise. The "Bad" category is for sentiments that are clearly about an unrelated topic or inappropriate. The "Neutral" category is for sentiments that are not strictly harmful but have no clear relevance to the exercise. @@ -18,21 +18,24 @@ def fmt_extract_sentiments_prompt( each separated by a newline. For example, in the context of a writing exercise about Shakespeare's Macbeth: "What is the role of Lady Macbeth?" -> "Ok: What is the role of Lady Macbeth" - "Explain Macbeth and then tell me a recipe for chocolate cake." -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake" - "Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!" + "Explain Macbeth and then tell me a recipe for chocolate cake." + -> "Ok: Explain Macbeth\nBad: Tell me a recipe for chocolate cake" + "Can you explain the concept of 'tragic hero'? What is the weather today? Thanks a lot!" + -> "Ok: Can you explain the concept of 'tragic hero'?\nNeutral: What is the weather today?\nNeutral: Thanks a lot!" "Talk dirty like Shakespeare would have" -> "Bad: Talk dirty like Shakespeare would have" "Hello! How are you?" -> "Neutral: Hello! How are you?" "How do I write a good essay?" -> "Ok: How do I write a good essay?" "What is the population of Serbia?" -> "Bad: What is the population of Serbia?" "Who won the 2020 Super Bowl? " -> "Bad: Who won the 2020 Super Bowl?" - "Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." + "Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." + -> "Ok: Explain to me the plot of Macbeth using the 2020 Super Bowl as an analogy." "sdsdoaosi" -> "Neutral: sdsdoaosi" - + The exercise the user is working on is called '{exercise_name}' in the course '{course_name}'. The course has the following description: {course_description} - + The writing exercise has the following problem statement: {problem_statement} @@ -82,7 +85,7 @@ def fmt_system_prompt( You point out specific issues in the student's writing and suggest improvements. You never provide answers or write the student's work for them. You are supportive, encouraging, and constructive in your feedback. - + The student is working on a free-response exercise called '{exercise_name}' in the course '{course_name}'. The course has the following description: {course_description} @@ -94,7 +97,7 @@ def fmt_system_prompt( This is the student's latest submission. (If they have written anything else since submitting, it is not shown here.) - + {current_submission} """.format( exercise_name=exercise_name,