From 4ebf3a85b13dc3b6132c918c2383f5ef8a6097ef Mon Sep 17 00:00:00 2001 From: Patrick Bassner Date: Fri, 19 Jul 2024 12:14:38 +0200 Subject: [PATCH] replace confidence with mastery --- app/pipeline/chat/course_chat_pipeline.py | 3 +-- app/pipeline/prompts/iris_course_chat_prompts.py | 2 +- app/pipeline/prompts/iris_course_chat_prompts_elicit.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/pipeline/chat/course_chat_pipeline.py b/app/pipeline/chat/course_chat_pipeline.py index 31dccc9a..42a046b0 100644 --- a/app/pipeline/chat/course_chat_pipeline.py +++ b/app/pipeline/chat/course_chat_pipeline.py @@ -232,7 +232,7 @@ def get_competency_list() -> list: regarding their progress overall or in a specific area. A competency has the following attributes: name, description, taxonomy, soft due date, optional, and mastery threshold. - The response may include metrics for each competency, such as progress and confidence (0%-100%). + The response may include metrics for each competency, such as progress and mastery (0%-100%). These are system-generated. The judgment of learning (JOL) values indicate the self-reported confidence by the student (0-5, 5 star). The object describing it also indicates the system-computed confidence at the time when the student @@ -248,7 +248,6 @@ def get_competency_list() -> list: "info": competency_metrics.competency_information.get(comp, None), "exercise_ids": competency_metrics.exercises.get(comp, []), "progress": competency_metrics.progress.get(comp, 0), - "confidence": competency_metrics.confidence.get(comp, 0), "mastery": ( (1 - weight) * competency_metrics.progress.get(comp, 0) + weight * competency_metrics.confidence.get(comp, 0) diff --git a/app/pipeline/prompts/iris_course_chat_prompts.py b/app/pipeline/prompts/iris_course_chat_prompts.py index d9ea310e..edee898b 100644 --- a/app/pipeline/prompts/iris_course_chat_prompts.py +++ b/app/pipeline/prompts/iris_course_chat_prompts.py @@ -88,7 +88,7 @@ tell_begin_agent_jol_prompt = """ Now, this time, the student did not send you a new message. You are being activated because something happened: the student submitted a JOL for a competency. -You should respond to this event. Focus on their self-reflection and the difference between their self-assessment and the system confidence value. +You should respond to this event. Focus on their self-reflection and the difference between their self-assessment and the system mastery value. Note that JoL goes from 0-5, where 0 is the lowest and 5 is the highest, while mastery goes from 0%-100%. If they ranked themselves lower than the system, you should encourage them to keep up the good work and that it is normal to feel less confident in the beginning. If they ranked themselves higher than the system, you should tell them that it is great to see that they are confident, but that they should keep in mind that the system has a more objective view of their progress diff --git a/app/pipeline/prompts/iris_course_chat_prompts_elicit.py b/app/pipeline/prompts/iris_course_chat_prompts_elicit.py index ef4cb221..03e07c2b 100644 --- a/app/pipeline/prompts/iris_course_chat_prompts_elicit.py +++ b/app/pipeline/prompts/iris_course_chat_prompts_elicit.py @@ -95,7 +95,7 @@ elicit_begin_agent_jol_prompt = """ Now, this time, the student did not send you a new message. You are being activated because something happened: the student submitted a JOL for a competency. -You should respond to this event. Focus on their self-reflection and the difference between their self-assessment and the system confidence value. +You should respond to this event. Focus on their self-reflection and the difference between their self-assessment and the system mastery value. Note that JoL goes from 0-5, where 0 is the lowest and 5 is the highest, while mastery goes from 0%-100%. If they ranked themselves lower than the system, you should encourage them to keep up the good work and that it is normal to feel less confident in the beginning. If they ranked themselves higher than the system, you should tell them that it is great to see that they are confident, but that they should keep in mind that the system has a more objective view of their progress