From f5105967dfe22c6cdafcbea454b8d7458288d8cd Mon Sep 17 00:00:00 2001 From: leon Date: Sat, 16 Nov 2024 19:30:21 +0100 Subject: [PATCH 01/13] Refactor feedback reference --- athena/athena/models/db_modeling_feedback.py | 4 +-- athena/athena/schemas/modeling_feedback.py | 6 ++-- .../apollon_json_transformer.py | 8 +++-- .../apollon_transformer/parser/uml_parser.py | 10 ++++++- .../models/assessment_model.py | 2 +- .../models/exercise_model.py | 3 +- .../utils/convert_to_athana_feedback_model.py | 29 ++++++++++++------- .../utils/get_exercise_model.py | 5 ++-- 8 files changed, 42 insertions(+), 25 deletions(-) diff --git a/athena/athena/models/db_modeling_feedback.py b/athena/athena/models/db_modeling_feedback.py index 142a59911..1fdffd23c 100644 --- a/athena/athena/models/db_modeling_feedback.py +++ b/athena/athena/models/db_modeling_feedback.py @@ -1,6 +1,6 @@ from typing import Optional -from sqlalchemy import Column, ForeignKey, JSON +from sqlalchemy import Column, ForeignKey, String from sqlalchemy.orm import relationship from athena.database import Base @@ -11,7 +11,7 @@ class DBModelingFeedback(DBFeedback, Base): __tablename__ = "modeling_feedbacks" - element_ids: Optional[list[str]] = Column(JSON) # type: ignore + reference: Optional[str] = Column(String, nullable=True) # type: ignore exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("modeling_exercises.id", ondelete="CASCADE"), index=True) submission_id = Column(BigIntegerWithAutoincrement, ForeignKey("modeling_submissions.id", ondelete="CASCADE"), index=True) diff --git a/athena/athena/schemas/modeling_feedback.py b/athena/athena/schemas/modeling_feedback.py index f77322f43..85245c2e0 100644 --- a/athena/athena/schemas/modeling_feedback.py +++ b/athena/athena/schemas/modeling_feedback.py @@ -1,11 +1,9 @@ -from typing import Optional, List - +from typing import Optional from pydantic import Field - from .feedback import Feedback class ModelingFeedback(Feedback): """Feedback on a modeling exercise.""" - element_ids: Optional[List[str]] = Field([], description="referenced diagram element IDs", example=["id_1"]) + reference: Optional[str] = Field(None, description="reference to the diagram element", example="ClassAttribute:5a337bdf-da00-4bd0-a6f0-78ba5b84330e") \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py index 3d67196df..7a487bfff 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py @@ -6,7 +6,7 @@ class ApollonJSONTransformer: @staticmethod - def transform_json(model: str) -> tuple[str, dict[str, str], str]: + def transform_json(model: str) -> tuple[str, dict[str, str], str, dict[str, str]]: """ Serialize a given Apollon diagram model to a string representation. This method converts the UML diagram model into a format similar to mermaid syntax, called "apollon". @@ -30,6 +30,8 @@ def transform_json(model: str) -> tuple[str, dict[str, str], str]: **{element['name']: element['id'] for element in parser.get_elements()}, **{relation['name']: relation['id'] for relation in parser.get_relations()} } - - return apollon_representation, names, diagram_type + + id_type_mapping = parser.get_id_to_type_mapping() + + return apollon_representation, names, diagram_type, id_type_mapping \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py index 57b9a63ad..97d7c43bb 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py @@ -93,4 +93,12 @@ def get_elements(self) -> List[Element]: return self.elements def get_relations(self) -> List[Relation]: - return self.relations \ No newline at end of file + return self.relations + + def get_id_to_type_mapping(self) -> Dict[str, str]: + id_to_type = {} + for element in self.elements: + id_to_type[element.id] = element.type + for relation in self.relations: + id_to_type[relation.id] = relation.type + return id_to_type \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py index 9ccc1933d..3468f7942 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py @@ -4,7 +4,7 @@ class FeedbackModel(BaseModel): title: str = Field(description="Very short title, i.e. feedback category or similar", example="Logic Error") description: str = Field(description="Feedback description") - element_names: Optional[List[str]] = Field(description="Referenced diagram element names, and relations (R) or empty if unreferenced") + element_name: Optional[str] = Field(description="Referenced diagram element, attribute names, and relations (R) or empty if unreferenced") credits: float = Field(0.0, description="Number of points received/deducted") grading_instruction_id: int = Field( description="ID of the grading instruction that was used to generate this feedback" diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py index 3a6f0672f..78a07a19c 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py @@ -11,4 +11,5 @@ class ExerciseModel(BaseModel): grading_instructions: Optional[str] = None submission_uml_type: str transformed_example_solution: Optional[str] = None - element_id_mapping: dict[str, str] \ No newline at end of file + element_id_mapping: dict[str, str] + id_type_mapping: dict[str, str] \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py index 06b4b5a66..dd44dfa71 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py @@ -11,33 +11,40 @@ def convert_to_athana_feedback_model( manual_structured_grading_instructions: Optional[List[GradingCriterion]] = None ) -> List[Feedback]: - grading_instruction_ids = set( + grading_instruction_ids = { grading_instruction.id - for criterion in manual_structured_grading_instructions or [] + for criterion in (manual_structured_grading_instructions or []) for grading_instruction in criterion.structured_grading_instructions - ) + } feedbacks = [] for feedback in feedback_result.feedbacks: # Each feedback has a grading_instruction_id. However we only want to have the grading_instruction_id in the final feedback that are associated with the manual structured grading instructions - grading_instruction_id = feedback.grading_instruction_id if feedback.grading_instruction_id in grading_instruction_ids else None - element_ids = [ - exercise_model.element_id_mapping[element] - for element in (feedback.element_names or []) - if element in exercise_model.element_id_mapping - ] + grading_instruction_id = ( + feedback.grading_instruction_id + if feedback.grading_instruction_id in grading_instruction_ids + else None + ) + + reference: Optional[str] = None + if feedback.element_name: + reference_id = exercise_model.element_id_mapping.get(feedback.element_name) + reference_type = exercise_model.id_type_mapping.get(reference_id) if reference_id else None + + if reference_type and reference_id: + reference = f"{reference_type}:{reference_id}" feedbacks.append(Feedback( exercise_id=exercise_model.exercise_id, submission_id=exercise_model.submission_id, title=feedback.title, description=feedback.description, - element_ids=element_ids, credits=feedback.credits, structured_grading_instruction_id=grading_instruction_id, meta={}, id=None, - is_graded=False + is_graded=False, + reference=reference )) return feedbacks \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/get_exercise_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/get_exercise_model.py index a2457f1f4..3489be738 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/get_exercise_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/get_exercise_model.py @@ -7,9 +7,9 @@ def get_exercise_model(exercise: Exercise, submission: Submission) -> ExerciseMo serialized_example_solution = None if exercise.example_solution: - serialized_example_solution, _, _ = ApollonJSONTransformer.transform_json(exercise.example_solution) + serialized_example_solution, _, _, _ = ApollonJSONTransformer.transform_json(exercise.example_solution) - transformed_submission, element_id_mapping, diagram_type = ApollonJSONTransformer.transform_json(submission.model) + transformed_submission, element_id_mapping, diagram_type, id_type_mapping = ApollonJSONTransformer.transform_json(submission.model) return ExerciseModel( submission_id=submission.id, @@ -22,6 +22,7 @@ def get_exercise_model(exercise: Exercise, submission: Submission) -> ExerciseMo submission_uml_type=diagram_type, transformed_example_solution=serialized_example_solution, element_id_mapping=element_id_mapping, + id_type_mapping=id_type_mapping ) From 6b98bb862aa3b2cdf4db57ffcb70c9578e2cbf2d Mon Sep 17 00:00:00 2001 From: leon Date: Mon, 18 Nov 2024 08:36:45 +0100 Subject: [PATCH 02/13] Enhance Apollon JSON transformer and parser for improved element ID mapping and uniqueness resolution --- .../apollon_json_transformer.py | 8 +-- .../apollon_transformer/parser/element.py | 61 +++++++++++++++++-- .../apollon_transformer/parser/uml_parser.py | 30 +++++++-- .../models/assessment_model.py | 4 +- .../models/exercise_model.py | 6 +- .../prompts/graded_feedback_prompt.py | 12 +++- .../utils/convert_to_athana_feedback_model.py | 9 ++- 7 files changed, 100 insertions(+), 30 deletions(-) diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py index 7a487bfff..2142a2cdb 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py @@ -25,11 +25,9 @@ def transform_json(model: str) -> tuple[str, dict[str, str], str, dict[str, str] # Convert the UML diagram to the apollon representation apollon_representation = parser.to_apollon() - # Extract elements and relations with their corresponding IDs and names - names = { - **{element['name']: element['id'] for element in parser.get_elements()}, - **{relation['name']: relation['id'] for relation in parser.get_relations()} - } + # Get the mapping of element, method, and attribute names to their corresponding IDs + # This is used to resolve references to as the apollon representation only contains names and not IDs + names = parser.get_element_id_mapping() id_type_mapping = parser.get_id_to_type_mapping() diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/element.py b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/element.py index 415b586a6..ef9c52a79 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/element.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/element.py @@ -1,4 +1,7 @@ -from typing import Dict, Any, List, Optional +# apollon_transformer/parser/element.py + +from typing import Dict, Any, List, Optional, Tuple +from string import ascii_uppercase class Element: """ @@ -17,15 +20,61 @@ def __init__(self, data: Dict[str, Any], element_dict: Optional[Dict[str, Any]] self.method_refs: List[str] = data.get('methods', []) self.attributes: List[str] = [] self.methods: List[str] = [] + self.attribute_id_mapping: Dict[str, str] = {} + self.method_id_mapping: Dict[str, str] = {} if element_dict is not None: self.resolve_references(element_dict) def resolve_references(self, element_dict: Dict[str, Any]): """ - Resolve attribute and method references using the provided element dictionary. The json data contains only references to other elements that represent attributes and methods. This method resolves these references to the actual names of the attributes and methods by looking up the corresponding elements via their IDs in the provided element dictionary. + Resolve attribute and method references using the provided element dictionary. + Ensures uniqueness among attribute and method names within the class. """ - self.attributes = [element_dict[ref].get("name", "") for ref in self.attribute_refs if ref in element_dict] - self.methods = [element_dict[ref].get('name', '') for ref in self.method_refs if ref in element_dict] + # Resolve attributes + self.attributes, self.attribute_id_mapping = self._resolve_uniqueness( + self.attribute_refs, element_dict) + + # Resolve methods + self.methods, self.method_id_mapping = self._resolve_uniqueness( + self.method_refs, element_dict) + + def _resolve_uniqueness( + self, refs: List[str], element_dict: Dict[str, Any] + ) -> Tuple[List[str], Dict[str, str]]: + name_counts: Dict[str, int] = {} + unique_full_names: List[str] = [] + id_mapping: Dict[str, str] = {} + for ref in refs: + if ref in element_dict: + full_name = element_dict[ref].get("name", "") + simplified_name = self.extract_name(full_name) + count = name_counts.get(simplified_name, 0) + if count > 0: + suffix = f"#{ascii_uppercase[count - 1]}" + simplified_name_with_suffix = f"{simplified_name}{suffix}" + else: + simplified_name_with_suffix = simplified_name + name_counts[simplified_name] = count + 1 + unique_full_names.append(full_name) + id_mapping[simplified_name_with_suffix] = ref + return unique_full_names, id_mapping + + @staticmethod + def extract_name(full_name: str) -> str: + """ + Extracts the simplified name from the full attribute or method name. + Removes visibility symbols, type annotations, and parameters. + """ + # Remove visibility symbols and leading/trailing spaces + name = full_name.lstrip('+-#~ ').strip() + # For attributes, split on ':' + if ':' in name: + name = name.split(':')[0].strip() + # For methods, split on '(' + elif '(' in name: + name = name.split('(')[0].strip() + return name + def to_apollon(self) -> str: parts = [f"[{self.type}] {self.name}"] @@ -41,6 +90,6 @@ def to_apollon(self) -> str: parts.append("{\n" + "\n".join(details) + "\n}") return " ".join(parts) - + def __getitem__(self, key): - return self.__dict__[key] \ No newline at end of file + return self.__dict__[key] diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py index 97d7c43bb..04b6ae6bc 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py @@ -94,11 +94,29 @@ def get_elements(self) -> List[Element]: def get_relations(self) -> List[Relation]: return self.relations - - def get_id_to_type_mapping(self) -> Dict[str, str]: - id_to_type = {} + + def get_element_id_mapping(self) -> Dict[str, str]: + """ + Creates a mapping from element names to their IDs, including attributes and methods. + """ + mapping = {} for element in self.elements: - id_to_type[element.id] = element.type + mapping[element.name] = element.id + for simplified_name_with_suffix, attr_id in element.attribute_id_mapping.items(): + mapping[f"{element.name}.{simplified_name_with_suffix}"] = attr_id + for simplified_name_with_suffix, method_id in element.method_id_mapping.items(): + mapping[f"{element.name}.{simplified_name_with_suffix}"] = method_id for relation in self.relations: - id_to_type[relation.id] = relation.type - return id_to_type \ No newline at end of file + mapping[relation.name] = relation.id + return mapping + + def get_id_to_type_mapping(self) -> Dict[str, str]: + """ + Creates a mapping from IDs to their types, including attributes and methods. + """ + mapping = {} + for element in self.data['elements'].values(): + mapping[element['id']] = element['type'] + for relation in self.data['relationships'].values(): + mapping[relation['id']] = relation['type'] + return mapping \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py index 3468f7942..29ccdb55a 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py @@ -1,10 +1,10 @@ -from typing import List, Optional, Sequence +from typing import Optional, Sequence from pydantic import BaseModel, Field class FeedbackModel(BaseModel): title: str = Field(description="Very short title, i.e. feedback category or similar", example="Logic Error") description: str = Field(description="Feedback description") - element_name: Optional[str] = Field(description="Referenced diagram element, attribute names, and relations (R) or empty if unreferenced") + element_name: Optional[str] = Field(description="Referenced diagram element, attribute names, and relations (use format: , ., ., R), or leave empty if unreferenced") credits: float = Field(0.0, description="Number of points received/deducted") grading_instruction_id: int = Field( description="ID of the grading instruction that was used to generate this feedback" diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py index 78a07a19c..1d9eef7c3 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, Dict from pydantic import BaseModel class ExerciseModel(BaseModel): @@ -11,5 +11,5 @@ class ExerciseModel(BaseModel): grading_instructions: Optional[str] = None submission_uml_type: str transformed_example_solution: Optional[str] = None - element_id_mapping: dict[str, str] - id_type_mapping: dict[str, str] \ No newline at end of file + element_id_mapping: Dict[str, str] + id_type_mapping: Dict[str, str] \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py b/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py index f2fa1390f..71662c84b 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py @@ -42,9 +42,15 @@ class GradedFeedbackInputs(BaseModel): {example_solution} Important: -Make sure to provide detailed feedback for each criterion. Always try to be as specific as possible. -Also make sure your feedback adds up to the correct number of points. If there are n points available and everything is correct, then the feedback should add up to n points. -Deeply think about the diagram and what the student potentially missed, misunderstood or mixed up. +- Make sure to provide detailed feedback for each criterion. Always try to be as specific as possible. +- Also make sure your feedback adds up to the correct number of points. If there are n points available and everything is correct, then the feedback should add up to n points. +- Deeply think about the diagram and what the student potentially missed, misunderstood, or mixed up. +- For the `element_name` field in the output, reference the specific diagram element, attribute, method, or relation related to the feedback. Use the following formats: + - For classes or elements: `` + - For attributes: `.` + - For methods: `.` + - For relations: `R` (e.g., `R1`, `R2`) +- If the feedback is not related to a specific element, leave the `element_name` field empty. The submission uses the following UML diagram format: diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py index dd44dfa71..fd96d3bbf 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py @@ -6,8 +6,8 @@ def convert_to_athana_feedback_model( - feedback_result : AssessmentModel, - exercise_model: ExerciseModel, + feedback_result: AssessmentModel, + exercise_model: ExerciseModel, manual_structured_grading_instructions: Optional[List[GradingCriterion]] = None ) -> List[Feedback]: @@ -19,10 +19,9 @@ def convert_to_athana_feedback_model( feedbacks = [] for feedback in feedback_result.feedbacks: - # Each feedback has a grading_instruction_id. However we only want to have the grading_instruction_id in the final feedback that are associated with the manual structured grading instructions grading_instruction_id = ( - feedback.grading_instruction_id - if feedback.grading_instruction_id in grading_instruction_ids + feedback.grading_instruction_id + if feedback.grading_instruction_id in grading_instruction_ids else None ) From 9de3f10f87b755b50b188b1826339770f9a09af0 Mon Sep 17 00:00:00 2001 From: leon Date: Wed, 27 Nov 2024 19:07:42 +0100 Subject: [PATCH 03/13] Add element_ids field to ModelingFeedback and update feedback conversion logic --- athena/athena/schemas/modeling_feedback.py | 3 ++- .../utils/convert_to_athana_feedback_model.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/athena/athena/schemas/modeling_feedback.py b/athena/athena/schemas/modeling_feedback.py index 85245c2e0..8b5123536 100644 --- a/athena/athena/schemas/modeling_feedback.py +++ b/athena/athena/schemas/modeling_feedback.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import List, Optional from pydantic import Field from .feedback import Feedback @@ -6,4 +6,5 @@ class ModelingFeedback(Feedback): """Feedback on a modeling exercise.""" + element_ids: Optional[List[str]] = Field([], description="referenced diagram element IDs", example=["id_1"]) reference: Optional[str] = Field(None, description="reference to the diagram element", example="ClassAttribute:5a337bdf-da00-4bd0-a6f0-78ba5b84330e") \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py index fd96d3bbf..0a6b260c5 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py @@ -43,7 +43,8 @@ def convert_to_athana_feedback_model( meta={}, id=None, is_graded=False, - reference=reference + reference=reference, + element_ids=[reference_id] if reference_id else [] )) return feedbacks \ No newline at end of file From 1e8e21da11fad847956dd3f6b5f9b9c7d6849bf1 Mon Sep 17 00:00:00 2001 From: leon Date: Thu, 28 Nov 2024 12:22:48 +0100 Subject: [PATCH 04/13] Add element_ids field to DBModelingFeedback model --- athena/athena/models/db_modeling_feedback.py | 1 + 1 file changed, 1 insertion(+) diff --git a/athena/athena/models/db_modeling_feedback.py b/athena/athena/models/db_modeling_feedback.py index 1fdffd23c..d121cfe98 100644 --- a/athena/athena/models/db_modeling_feedback.py +++ b/athena/athena/models/db_modeling_feedback.py @@ -11,6 +11,7 @@ class DBModelingFeedback(DBFeedback, Base): __tablename__ = "modeling_feedbacks" + element_ids: Optional[list[str]] = Column(JSON) # type: ignore reference: Optional[str] = Column(String, nullable=True) # type: ignore exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("modeling_exercises.id", ondelete="CASCADE"), index=True) From a5203ff5c528b44f1202c03e6cfd016d5a39543d Mon Sep 17 00:00:00 2001 From: leon Date: Fri, 29 Nov 2024 18:28:04 +0100 Subject: [PATCH 05/13] Add JSON type import to db_modeling_feedback.py --- athena/athena/models/db_modeling_feedback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/athena/athena/models/db_modeling_feedback.py b/athena/athena/models/db_modeling_feedback.py index d121cfe98..47d3a636e 100644 --- a/athena/athena/models/db_modeling_feedback.py +++ b/athena/athena/models/db_modeling_feedback.py @@ -1,6 +1,6 @@ from typing import Optional -from sqlalchemy import Column, ForeignKey, String +from sqlalchemy import Column, ForeignKey, JSON, String from sqlalchemy.orm import relationship from athena.database import Base From 7b488a36cd0952e42f21514543e71c516f7ea6b6 Mon Sep 17 00:00:00 2001 From: leon Date: Tue, 3 Dec 2024 13:26:49 +0100 Subject: [PATCH 06/13] add structured grading instruction cache --- athena/athena/models/__init__.py | 3 +- .../models/db_structured_grading_criterion.py | 17 ++++++++++ athena/athena/schemas/__init__.py | 3 +- athena/athena/schemas/grading_criterion.py | 5 +-- .../schemas/structured_grading_criterion.py | 7 ++++ .../structured_grading_criterion_storage.py | 32 ++++++++++++++++++ .../core/generate_suggestions.py | 2 +- .../get_structured_grading_instructions.py | 33 +++++++++++++++++-- 8 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 athena/athena/models/db_structured_grading_criterion.py create mode 100644 athena/athena/schemas/structured_grading_criterion.py create mode 100644 athena/athena/storage/structured_grading_criterion_storage.py diff --git a/athena/athena/models/__init__.py b/athena/athena/models/__init__.py index 5febe42a6..8f63cf88c 100644 --- a/athena/athena/models/__init__.py +++ b/athena/athena/models/__init__.py @@ -14,4 +14,5 @@ from .db_modeling_submission import DBModelingSubmission from .db_programming_feedback import DBProgrammingFeedback from .db_text_feedback import DBTextFeedback -from .db_modeling_feedback import DBModelingFeedback \ No newline at end of file +from .db_modeling_feedback import DBModelingFeedback +from .db_structured_grading_criterion import DBStructuredGradingCriterion \ No newline at end of file diff --git a/athena/athena/models/db_structured_grading_criterion.py b/athena/athena/models/db_structured_grading_criterion.py new file mode 100644 index 000000000..897e54c69 --- /dev/null +++ b/athena/athena/models/db_structured_grading_criterion.py @@ -0,0 +1,17 @@ +from sqlalchemy import Column, JSON, String, ForeignKey +from sqlalchemy.orm import relationship + +from athena.database import Base +from .big_integer_with_autoincrement import BigIntegerWithAutoincrement + + +class DBStructuredGradingCriterion(Base): + __tablename__ = "structured_grading_criterion" + id = Column(BigIntegerWithAutoincrement, primary_key=True, index=True, + autoincrement=True) + exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("exercises.id", ondelete="CASCADE"), index=True, unique=True) # Only one cached instruction per exercise + instructions_hash = Column(String, nullable=False) + structured_grading_criterion = Column(JSON, nullable=False) + lms_url = Column(String, nullable=False) + + exercise = relationship("DBExercise", back_populates="structured_grading_criterion") \ No newline at end of file diff --git a/athena/athena/schemas/__init__.py b/athena/athena/schemas/__init__.py index 244c2d9c0..b5f6a7943 100644 --- a/athena/athena/schemas/__init__.py +++ b/athena/athena/schemas/__init__.py @@ -13,4 +13,5 @@ from .modeling_feedback import ModelingFeedback from .modeling_exercise import ModelingExercise from .modeling_submission import ModelingSubmission -from .grading_criterion import GradingCriterion, StructuredGradingInstruction, StructuredGradingCriterion \ No newline at end of file +from .grading_criterion import GradingCriterion, StructuredGradingInstruction +from .structured_grading_criterion import StructuredGradingCriterion \ No newline at end of file diff --git a/athena/athena/schemas/grading_criterion.py b/athena/athena/schemas/grading_criterion.py index 15225a7dc..186478830 100644 --- a/athena/athena/schemas/grading_criterion.py +++ b/athena/athena/schemas/grading_criterion.py @@ -23,7 +23,4 @@ class GradingCriterion(Schema, ABC): title: Optional[str] = Field(None, example="Some instructions") structured_grading_instructions: List[StructuredGradingInstruction] = Field( [], example=[{"credits": 1.0, "gradingScale": "Good", "instructionDescription": "Some instructions", "feedback": "Nicely done!", "usageCount": 1}, - {"credits": 0.0, "gradingScale": "Bad", "instructionDescription": "Some instructions", "feedback": "Try again!", "usageCount": 0}]) - -class StructuredGradingCriterion(BaseModel): - criteria: List[GradingCriterion] \ No newline at end of file + {"credits": 0.0, "gradingScale": "Bad", "instructionDescription": "Some instructions", "feedback": "Try again!", "usageCount": 0}]) \ No newline at end of file diff --git a/athena/athena/schemas/structured_grading_criterion.py b/athena/athena/schemas/structured_grading_criterion.py new file mode 100644 index 000000000..89f1beb7b --- /dev/null +++ b/athena/athena/schemas/structured_grading_criterion.py @@ -0,0 +1,7 @@ +from typing import List +from athena.schemas.grading_criterion import GradingCriterion +from pydantic import BaseModel + + +class StructuredGradingCriterion(BaseModel): + criteria: List[GradingCriterion] \ No newline at end of file diff --git a/athena/athena/storage/structured_grading_criterion_storage.py b/athena/athena/storage/structured_grading_criterion_storage.py new file mode 100644 index 000000000..0abd1b3f2 --- /dev/null +++ b/athena/athena/storage/structured_grading_criterion_storage.py @@ -0,0 +1,32 @@ +from typing import Optional +from athena.contextvars import get_lms_url +from athena.database import get_db + +from athena.models import DBStructuredGradingCriterion +from athena.schemas import StructuredGradingCriterion + +def get_structured_grading_criterion(exercise_id: int, current_hash: Optional[str] = None) -> Optional[StructuredGradingCriterion]: + lms_url = get_lms_url() + with get_db() as db: + cache_entry = db.query(DBStructuredGradingCriterion).filter_by( + exercise_id=exercise_id, lms_url=lms_url + ).first() + if cache_entry is not None and (current_hash is None or cache_entry.instructions_hash.is_(current_hash)): + return StructuredGradingCriterion.parse_raw(cache_entry.cached_instructions) + + return None + +def store_structured_grading_criterion( + exercise_id: int, hash: str, structured_instructions: StructuredGradingCriterion +): + lms_url = get_lms_url() + with get_db() as db: + db.merge( + DBStructuredGradingCriterion( + exercise_id=exercise_id, + lms_url=lms_url, + instructions_hash=hash, + cached_instructions=structured_instructions.json(), + ) + ) + db.commit() \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py index 6db69e7d2..61f051348 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py @@ -1,4 +1,4 @@ -from athena.schemas.grading_criterion import StructuredGradingCriterion +from athena.schemas.structured_grading_criterion import StructuredGradingCriterion from langchain_core.output_parsers import PydanticOutputParser from langchain_core.prompts import ChatPromptTemplate diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py index ae84e0ddf..4796ae01a 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py @@ -1,9 +1,13 @@ -from typing import List, Optional +import hashlib +import json +from typing import Any, Dict, List, Optional +from athena import logger from athena.metadata import emit_meta +from athena.storage.structured_grading_criterion_storage import get_structured_grading_criterion, store_structured_grading_criterion from langchain_core.output_parsers import PydanticOutputParser from langchain_core.prompts import ChatPromptTemplate -from athena.schemas.grading_criterion import GradingCriterion, StructuredGradingCriterion +from athena.schemas import GradingCriterion, StructuredGradingCriterion from llm_core.utils.predict_and_parse import predict_and_parse from module_modeling_llm.config import BasicApproachConfig from module_modeling_llm.models.exercise_model import ExerciseModel @@ -19,6 +23,12 @@ async def get_structured_grading_instructions( if grading_criteria: return StructuredGradingCriterion(criteria=grading_criteria) + + # Check if we have cached instructions for this exercise + current_hash = get_grading_instructions_hash(exercise_model) + cached_instructions = get_structured_grading_criterion(exercise_model.exercise_id, current_hash) + if cached_instructions: + return cached_instructions chat_prompt = ChatPromptTemplate.from_messages([ ("system", config.generate_suggestions_prompt.structured_grading_instructions_system_message), @@ -53,5 +63,22 @@ async def get_structured_grading_instructions( if not grading_instruction_result: raise ValueError("No structured grading instructions were returned by the model.") + + # Cache the grading instructions + hash = get_grading_instructions_hash(exercise_model) + store_structured_grading_criterion(exercise_model.exercise_id, hash, grading_instruction_result) + + return grading_instruction_result + +def get_grading_instructions_hash(exercise: ExerciseModel) -> str: + + hashable_data = { + "problem_statement": exercise.problem_statement, + "grading_instructions": exercise.grading_instructions, + "sample_solution": exercise.transformed_example_solution, + "max_points": exercise.max_points, + "bonus_points": exercise.bonus_points, + } - return grading_instruction_result \ No newline at end of file + json_string = json.dumps(hashable_data, sort_keys=True, default=str) + return hashlib.sha256(json_string.encode()).hexdigest() \ No newline at end of file From f2604c9b12e0ac3264c4c6276ce9166ad5cf85ad Mon Sep 17 00:00:00 2001 From: leon Date: Tue, 3 Dec 2024 17:14:36 +0100 Subject: [PATCH 07/13] Increase default max_tokens to 4000 in OpenAI model configuration --- llm_core/llm_core/models/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_core/llm_core/models/openai.py b/llm_core/llm_core/models/openai.py index d12df22d4..7bcc0f11f 100644 --- a/llm_core/llm_core/models/openai.py +++ b/llm_core/llm_core/models/openai.py @@ -67,7 +67,7 @@ class OpenAIModelConfig(ModelConfig): model_name: OpenAIModel = Field(default=default_openai_model, # type: ignore description="The name of the model to use.") - max_tokens: PositiveInt = Field(1000, description="""\ + max_tokens: PositiveInt = Field(4000, description="""\ The maximum number of [tokens](https://platform.openai.com/tokenizer) to generate in the chat completion. The total length of input tokens and generated tokens is limited by the model's context length. \ From 87a66a733b5e58b3301a54d0e1c709fc441ae4c6 Mon Sep 17 00:00:00 2001 From: leon Date: Tue, 3 Dec 2024 18:38:38 +0100 Subject: [PATCH 08/13] Increase max_input_tokens to 5000 and update element_ids assignment in feedback model conversion --- .../modeling/module_modeling_llm/module_modeling_llm/config.py | 2 +- .../utils/convert_to_athana_feedback_model.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/config.py b/modules/modeling/module_modeling_llm/module_modeling_llm/config.py index 169ee047a..c7f0f4ad0 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/config.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/config.py @@ -43,7 +43,7 @@ class GenerateSuggestionsPrompt(BaseModel): class BasicApproachConfig(BaseModel): """This approach uses a LLM with a single prompt to generate feedback in a single step.""" - max_input_tokens: int = Field(default=3000, description="Maximum number of tokens in the input prompt.") + max_input_tokens: int = Field(default=5000, description="Maximum number of tokens in the input prompt.") model: ModelConfigType = Field(default=DefaultModelConfig()) # type: ignore generate_suggestions_prompt: GenerateSuggestionsPrompt = Field(default=GenerateSuggestionsPrompt()) diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py index 0a6b260c5..4ade6b3c7 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py @@ -44,7 +44,7 @@ def convert_to_athana_feedback_model( id=None, is_graded=False, reference=reference, - element_ids=[reference_id] if reference_id else [] + element_ids=[reference] if reference else [] # Todo: Remove after adding migrations to athena )) return feedbacks \ No newline at end of file From 322fd5567ab61a67b03f06f0b1ffd11b3b5ec9af Mon Sep 17 00:00:00 2001 From: leon Date: Fri, 6 Dec 2024 10:40:58 +0100 Subject: [PATCH 09/13] Refactor exercise models to implement polymorphism and establish relationships; fix foreign key references and ensure proper inheritance structure. --- athena/athena/app.py | 2 +- athena/athena/models/db_exercise.py | 17 ++++++++++++++--- athena/athena/models/db_modeling_exercise.py | 19 +++++++++++-------- .../athena/models/db_programming_exercise.py | 12 +++++++++--- .../models/db_structured_grading_criterion.py | 2 +- athena/athena/models/db_text_exercise.py | 12 +++++++++--- athena/athena/storage/exercise_storage.py | 3 +++ 7 files changed, 48 insertions(+), 19 deletions(-) diff --git a/athena/athena/app.py b/athena/athena/app.py index b73cc045d..e9d653f03 100644 --- a/athena/athena/app.py +++ b/athena/athena/app.py @@ -69,4 +69,4 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE return JSONResponse( status_code=422, content={"detail": exc.errors()}, - ) + ) \ No newline at end of file diff --git a/athena/athena/models/db_exercise.py b/athena/athena/models/db_exercise.py index 13fd5593e..d0c5e4909 100644 --- a/athena/athena/models/db_exercise.py +++ b/athena/athena/models/db_exercise.py @@ -1,18 +1,29 @@ from sqlalchemy import Column, String, Float, JSON, Enum as SqlEnum - +from sqlalchemy.orm import relationship +from athena.database import Base from athena.schemas import ExerciseType from .model import Model from .big_integer_with_autoincrement import BigIntegerWithAutoincrement -class DBExercise(Model): +class DBExercise(Model, Base): + __tablename__ = "exercise" id = Column(BigIntegerWithAutoincrement, primary_key=True, index=True, nullable=False) lms_url = Column(String, index=True, nullable=False) title = Column(String, index=True, nullable=False) - type = Column(SqlEnum(ExerciseType), index=True, nullable=False) max_points = Column(Float, index=True, nullable=False) bonus_points = Column(Float, index=True, nullable=False) grading_instructions = Column(String) problem_statement = Column(String) grading_criteria = Column(JSON, nullable=True) meta = Column(JSON, nullable=False) + + structured_grading_criterion = relationship("DBStructuredGradingCriterion", back_populates="exercise", uselist=False) # Use uselist=False for one-to-one + + # Polymorphism, discriminator attribute + type = Column(SqlEnum(ExerciseType), index=True, nullable=False) + + __mapper_args__ = { + 'polymorphic_identity': 'exercise', + 'polymorphic_on': 'type' + } \ No newline at end of file diff --git a/athena/athena/models/db_modeling_exercise.py b/athena/athena/models/db_modeling_exercise.py index 450b32e77..818724ab8 100644 --- a/athena/athena/models/db_modeling_exercise.py +++ b/athena/athena/models/db_modeling_exercise.py @@ -1,14 +1,17 @@ -from sqlalchemy import Column, String +from athena.schemas.exercise_type import ExerciseType +from sqlalchemy import Column, String, ForeignKey from sqlalchemy.orm import relationship - -from athena.database import Base from .db_exercise import DBExercise - - -class DBModelingExercise(DBExercise, Base): +from .big_integer_with_autoincrement import BigIntegerWithAutoincrement +class DBModelingExercise(DBExercise): __tablename__ = "modeling_exercises" - example_solution: str = Column(String) # type: ignore - + example_solution = Column(String) # type: ignore + + id = Column(BigIntegerWithAutoincrement, ForeignKey('exercise.id'), primary_key=True) submissions = relationship("DBModelingSubmission", back_populates="exercise") feedbacks = relationship("DBModelingFeedback", back_populates="exercise") + + __mapper_args__ = { + 'polymorphic_identity': ExerciseType.modeling.value + } \ No newline at end of file diff --git a/athena/athena/models/db_programming_exercise.py b/athena/athena/models/db_programming_exercise.py index b558dd58f..3060e5831 100644 --- a/athena/athena/models/db_programming_exercise.py +++ b/athena/athena/models/db_programming_exercise.py @@ -1,13 +1,15 @@ -from sqlalchemy import Column, String +from athena.schemas.exercise_type import ExerciseType +from sqlalchemy import Column, String, ForeignKey from sqlalchemy.orm import relationship -from athena.database import Base from .db_exercise import DBExercise +from .big_integer_with_autoincrement import BigIntegerWithAutoincrement -class DBProgrammingExercise(DBExercise, Base): +class DBProgrammingExercise(DBExercise): __tablename__ = "programming_exercises" + id = Column(BigIntegerWithAutoincrement, ForeignKey('exercise.id'), primary_key=True) programming_language: str = Column(String, nullable=False) # type: ignore solution_repository_uri: str = Column(String, nullable=False) # type: ignore template_repository_uri: str = Column(String, nullable=False) # type: ignore @@ -15,3 +17,7 @@ class DBProgrammingExercise(DBExercise, Base): submissions = relationship("DBProgrammingSubmission", back_populates="exercise") feedbacks = relationship("DBProgrammingFeedback", back_populates="exercise") + + __mapper_args__ = { + 'polymorphic_identity': ExerciseType.programming.value + } \ No newline at end of file diff --git a/athena/athena/models/db_structured_grading_criterion.py b/athena/athena/models/db_structured_grading_criterion.py index 897e54c69..b0679ae70 100644 --- a/athena/athena/models/db_structured_grading_criterion.py +++ b/athena/athena/models/db_structured_grading_criterion.py @@ -9,7 +9,7 @@ class DBStructuredGradingCriterion(Base): __tablename__ = "structured_grading_criterion" id = Column(BigIntegerWithAutoincrement, primary_key=True, index=True, autoincrement=True) - exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("exercises.id", ondelete="CASCADE"), index=True, unique=True) # Only one cached instruction per exercise + exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("exercise.id", ondelete="CASCADE"), index=True, unique=True) # Only one cached instruction per exercise instructions_hash = Column(String, nullable=False) structured_grading_criterion = Column(JSON, nullable=False) lms_url = Column(String, nullable=False) diff --git a/athena/athena/models/db_text_exercise.py b/athena/athena/models/db_text_exercise.py index f3c2823ef..05df9e5c0 100644 --- a/athena/athena/models/db_text_exercise.py +++ b/athena/athena/models/db_text_exercise.py @@ -1,14 +1,20 @@ -from sqlalchemy import Column, String +from athena.schemas.exercise_type import ExerciseType +from sqlalchemy import Column, String, ForeignKey from sqlalchemy.orm import relationship -from athena.database import Base from .db_exercise import DBExercise +from .big_integer_with_autoincrement import BigIntegerWithAutoincrement -class DBTextExercise(DBExercise, Base): +class DBTextExercise(DBExercise): __tablename__ = "text_exercises" + id = Column(BigIntegerWithAutoincrement, ForeignKey('exercise.id'), primary_key=True) example_solution: str = Column(String) # type: ignore submissions = relationship("DBTextSubmission", back_populates="exercise") feedbacks = relationship("DBTextFeedback", back_populates="exercise") + + __mapper_args__ = { + 'polymorphic_identity': ExerciseType.text.value + } \ No newline at end of file diff --git a/athena/athena/storage/exercise_storage.py b/athena/athena/storage/exercise_storage.py index 77ba205ca..f452fc00c 100644 --- a/athena/athena/storage/exercise_storage.py +++ b/athena/athena/storage/exercise_storage.py @@ -42,10 +42,13 @@ def store_exercises(exercises: List[Exercise], lms_url: Optional[str] = None): if lms_url is None: lms_url = get_lms_url() + print("Exercise lms_url 2: ", lms_url, "\n\n\n\n") + with get_db() as db: for e in exercises: exercise_model = e.to_model() exercise_model.lms_url = lms_url + print("Exercise model: ", exercise_model.lms_url, "\n\n\n\n") db.merge(exercise_model) db.commit() From a83ac8b5f9d5e181d451cbb23662da7c2601cfae Mon Sep 17 00:00:00 2001 From: leon Date: Fri, 6 Dec 2024 11:10:07 +0100 Subject: [PATCH 10/13] Refactor exercise storage and structured grading criterion handling; remove debug prints, update caching logic, and change serialization method for structured grading instructions --- athena/athena/storage/exercise_storage.py | 3 --- .../structured_grading_criterion_storage.py | 19 +++++++++---------- .../get_structured_grading_instructions.py | 1 + 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/athena/athena/storage/exercise_storage.py b/athena/athena/storage/exercise_storage.py index f452fc00c..77ba205ca 100644 --- a/athena/athena/storage/exercise_storage.py +++ b/athena/athena/storage/exercise_storage.py @@ -42,13 +42,10 @@ def store_exercises(exercises: List[Exercise], lms_url: Optional[str] = None): if lms_url is None: lms_url = get_lms_url() - print("Exercise lms_url 2: ", lms_url, "\n\n\n\n") - with get_db() as db: for e in exercises: exercise_model = e.to_model() exercise_model.lms_url = lms_url - print("Exercise model: ", exercise_model.lms_url, "\n\n\n\n") db.merge(exercise_model) db.commit() diff --git a/athena/athena/storage/structured_grading_criterion_storage.py b/athena/athena/storage/structured_grading_criterion_storage.py index 0abd1b3f2..459e4c02d 100644 --- a/athena/athena/storage/structured_grading_criterion_storage.py +++ b/athena/athena/storage/structured_grading_criterion_storage.py @@ -6,15 +6,14 @@ from athena.schemas import StructuredGradingCriterion def get_structured_grading_criterion(exercise_id: int, current_hash: Optional[str] = None) -> Optional[StructuredGradingCriterion]: - lms_url = get_lms_url() - with get_db() as db: - cache_entry = db.query(DBStructuredGradingCriterion).filter_by( - exercise_id=exercise_id, lms_url=lms_url - ).first() - if cache_entry is not None and (current_hash is None or cache_entry.instructions_hash.is_(current_hash)): - return StructuredGradingCriterion.parse_raw(cache_entry.cached_instructions) - - return None + lms_url = get_lms_url() + with get_db() as db: + cache_entry = db.query(DBStructuredGradingCriterion).filter_by( + exercise_id=exercise_id, lms_url=lms_url + ).first() + if cache_entry is not None and (current_hash is None or cache_entry.instructions_hash == current_hash): # type: ignore + return StructuredGradingCriterion.parse_obj(cache_entry.structured_grading_criterion) + return None def store_structured_grading_criterion( exercise_id: int, hash: str, structured_instructions: StructuredGradingCriterion @@ -26,7 +25,7 @@ def store_structured_grading_criterion( exercise_id=exercise_id, lms_url=lms_url, instructions_hash=hash, - cached_instructions=structured_instructions.json(), + structured_grading_criterion=structured_instructions.dict(), ) ) db.commit() \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py index 4796ae01a..75b7ded47 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py @@ -28,6 +28,7 @@ async def get_structured_grading_instructions( current_hash = get_grading_instructions_hash(exercise_model) cached_instructions = get_structured_grading_criterion(exercise_model.exercise_id, current_hash) if cached_instructions: + print("Using cached instructions") return cached_instructions chat_prompt = ChatPromptTemplate.from_messages([ From 0eff1183e4fd4d926d01d88bc5d11d1313769953 Mon Sep 17 00:00:00 2001 From: leon Date: Sat, 7 Dec 2024 12:27:59 +0100 Subject: [PATCH 11/13] Remove DBStructuredGradingCriterion model and clean up imports in __init__.py --- athena/athena/models/__init__.py | 3 +-- .../models/db_structured_grading_criterion.py | 17 ----------------- 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 athena/athena/models/db_structured_grading_criterion.py diff --git a/athena/athena/models/__init__.py b/athena/athena/models/__init__.py index 8f63cf88c..5febe42a6 100644 --- a/athena/athena/models/__init__.py +++ b/athena/athena/models/__init__.py @@ -14,5 +14,4 @@ from .db_modeling_submission import DBModelingSubmission from .db_programming_feedback import DBProgrammingFeedback from .db_text_feedback import DBTextFeedback -from .db_modeling_feedback import DBModelingFeedback -from .db_structured_grading_criterion import DBStructuredGradingCriterion \ No newline at end of file +from .db_modeling_feedback import DBModelingFeedback \ No newline at end of file diff --git a/athena/athena/models/db_structured_grading_criterion.py b/athena/athena/models/db_structured_grading_criterion.py deleted file mode 100644 index b0679ae70..000000000 --- a/athena/athena/models/db_structured_grading_criterion.py +++ /dev/null @@ -1,17 +0,0 @@ -from sqlalchemy import Column, JSON, String, ForeignKey -from sqlalchemy.orm import relationship - -from athena.database import Base -from .big_integer_with_autoincrement import BigIntegerWithAutoincrement - - -class DBStructuredGradingCriterion(Base): - __tablename__ = "structured_grading_criterion" - id = Column(BigIntegerWithAutoincrement, primary_key=True, index=True, - autoincrement=True) - exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("exercise.id", ondelete="CASCADE"), index=True, unique=True) # Only one cached instruction per exercise - instructions_hash = Column(String, nullable=False) - structured_grading_criterion = Column(JSON, nullable=False) - lms_url = Column(String, nullable=False) - - exercise = relationship("DBExercise", back_populates="structured_grading_criterion") \ No newline at end of file From 672f76210bbe177776cac4ff13a8be315384b4bf Mon Sep 17 00:00:00 2001 From: leon Date: Sat, 7 Dec 2024 12:39:56 +0100 Subject: [PATCH 12/13] Refactor grading criterion handling: remove structured grading criterion model, update imports --- athena/athena/app.py | 2 +- athena/athena/models/db_exercise.py | 2 -- athena/athena/schemas/__init__.py | 3 +- athena/athena/schemas/grading_criterion.py | 5 ++- .../schemas/structured_grading_criterion.py | 7 ---- .../structured_grading_criterion_storage.py | 31 ----------------- .../core/generate_suggestions.py | 2 +- .../get_structured_grading_instructions.py | 34 ++----------------- 8 files changed, 10 insertions(+), 76 deletions(-) delete mode 100644 athena/athena/schemas/structured_grading_criterion.py delete mode 100644 athena/athena/storage/structured_grading_criterion_storage.py diff --git a/athena/athena/app.py b/athena/athena/app.py index e9d653f03..b73cc045d 100644 --- a/athena/athena/app.py +++ b/athena/athena/app.py @@ -69,4 +69,4 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE return JSONResponse( status_code=422, content={"detail": exc.errors()}, - ) \ No newline at end of file + ) diff --git a/athena/athena/models/db_exercise.py b/athena/athena/models/db_exercise.py index d0c5e4909..323cc5527 100644 --- a/athena/athena/models/db_exercise.py +++ b/athena/athena/models/db_exercise.py @@ -18,8 +18,6 @@ class DBExercise(Model, Base): grading_criteria = Column(JSON, nullable=True) meta = Column(JSON, nullable=False) - structured_grading_criterion = relationship("DBStructuredGradingCriterion", back_populates="exercise", uselist=False) # Use uselist=False for one-to-one - # Polymorphism, discriminator attribute type = Column(SqlEnum(ExerciseType), index=True, nullable=False) diff --git a/athena/athena/schemas/__init__.py b/athena/athena/schemas/__init__.py index b5f6a7943..244c2d9c0 100644 --- a/athena/athena/schemas/__init__.py +++ b/athena/athena/schemas/__init__.py @@ -13,5 +13,4 @@ from .modeling_feedback import ModelingFeedback from .modeling_exercise import ModelingExercise from .modeling_submission import ModelingSubmission -from .grading_criterion import GradingCriterion, StructuredGradingInstruction -from .structured_grading_criterion import StructuredGradingCriterion \ No newline at end of file +from .grading_criterion import GradingCriterion, StructuredGradingInstruction, StructuredGradingCriterion \ No newline at end of file diff --git a/athena/athena/schemas/grading_criterion.py b/athena/athena/schemas/grading_criterion.py index 186478830..6d0d2f752 100644 --- a/athena/athena/schemas/grading_criterion.py +++ b/athena/athena/schemas/grading_criterion.py @@ -23,4 +23,7 @@ class GradingCriterion(Schema, ABC): title: Optional[str] = Field(None, example="Some instructions") structured_grading_instructions: List[StructuredGradingInstruction] = Field( [], example=[{"credits": 1.0, "gradingScale": "Good", "instructionDescription": "Some instructions", "feedback": "Nicely done!", "usageCount": 1}, - {"credits": 0.0, "gradingScale": "Bad", "instructionDescription": "Some instructions", "feedback": "Try again!", "usageCount": 0}]) \ No newline at end of file + {"credits": 0.0, "gradingScale": "Bad", "instructionDescription": "Some instructions", "feedback": "Try again!", "usageCount": 0}]) + +class StructuredGradingCriterion(BaseModel): + criteria: List[GradingCriterion] \ No newline at end of file diff --git a/athena/athena/schemas/structured_grading_criterion.py b/athena/athena/schemas/structured_grading_criterion.py deleted file mode 100644 index 89f1beb7b..000000000 --- a/athena/athena/schemas/structured_grading_criterion.py +++ /dev/null @@ -1,7 +0,0 @@ -from typing import List -from athena.schemas.grading_criterion import GradingCriterion -from pydantic import BaseModel - - -class StructuredGradingCriterion(BaseModel): - criteria: List[GradingCriterion] \ No newline at end of file diff --git a/athena/athena/storage/structured_grading_criterion_storage.py b/athena/athena/storage/structured_grading_criterion_storage.py deleted file mode 100644 index 459e4c02d..000000000 --- a/athena/athena/storage/structured_grading_criterion_storage.py +++ /dev/null @@ -1,31 +0,0 @@ -from typing import Optional -from athena.contextvars import get_lms_url -from athena.database import get_db - -from athena.models import DBStructuredGradingCriterion -from athena.schemas import StructuredGradingCriterion - -def get_structured_grading_criterion(exercise_id: int, current_hash: Optional[str] = None) -> Optional[StructuredGradingCriterion]: - lms_url = get_lms_url() - with get_db() as db: - cache_entry = db.query(DBStructuredGradingCriterion).filter_by( - exercise_id=exercise_id, lms_url=lms_url - ).first() - if cache_entry is not None and (current_hash is None or cache_entry.instructions_hash == current_hash): # type: ignore - return StructuredGradingCriterion.parse_obj(cache_entry.structured_grading_criterion) - return None - -def store_structured_grading_criterion( - exercise_id: int, hash: str, structured_instructions: StructuredGradingCriterion -): - lms_url = get_lms_url() - with get_db() as db: - db.merge( - DBStructuredGradingCriterion( - exercise_id=exercise_id, - lms_url=lms_url, - instructions_hash=hash, - structured_grading_criterion=structured_instructions.dict(), - ) - ) - db.commit() \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py index 61f051348..6db69e7d2 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py @@ -1,4 +1,4 @@ -from athena.schemas.structured_grading_criterion import StructuredGradingCriterion +from athena.schemas.grading_criterion import StructuredGradingCriterion from langchain_core.output_parsers import PydanticOutputParser from langchain_core.prompts import ChatPromptTemplate diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py index 75b7ded47..ae84e0ddf 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py @@ -1,13 +1,9 @@ -import hashlib -import json -from typing import Any, Dict, List, Optional -from athena import logger +from typing import List, Optional from athena.metadata import emit_meta -from athena.storage.structured_grading_criterion_storage import get_structured_grading_criterion, store_structured_grading_criterion from langchain_core.output_parsers import PydanticOutputParser from langchain_core.prompts import ChatPromptTemplate -from athena.schemas import GradingCriterion, StructuredGradingCriterion +from athena.schemas.grading_criterion import GradingCriterion, StructuredGradingCriterion from llm_core.utils.predict_and_parse import predict_and_parse from module_modeling_llm.config import BasicApproachConfig from module_modeling_llm.models.exercise_model import ExerciseModel @@ -23,13 +19,6 @@ async def get_structured_grading_instructions( if grading_criteria: return StructuredGradingCriterion(criteria=grading_criteria) - - # Check if we have cached instructions for this exercise - current_hash = get_grading_instructions_hash(exercise_model) - cached_instructions = get_structured_grading_criterion(exercise_model.exercise_id, current_hash) - if cached_instructions: - print("Using cached instructions") - return cached_instructions chat_prompt = ChatPromptTemplate.from_messages([ ("system", config.generate_suggestions_prompt.structured_grading_instructions_system_message), @@ -64,22 +53,5 @@ async def get_structured_grading_instructions( if not grading_instruction_result: raise ValueError("No structured grading instructions were returned by the model.") - - # Cache the grading instructions - hash = get_grading_instructions_hash(exercise_model) - store_structured_grading_criterion(exercise_model.exercise_id, hash, grading_instruction_result) - - return grading_instruction_result - -def get_grading_instructions_hash(exercise: ExerciseModel) -> str: - - hashable_data = { - "problem_statement": exercise.problem_statement, - "grading_instructions": exercise.grading_instructions, - "sample_solution": exercise.transformed_example_solution, - "max_points": exercise.max_points, - "bonus_points": exercise.bonus_points, - } - json_string = json.dumps(hashable_data, sort_keys=True, default=str) - return hashlib.sha256(json_string.encode()).hexdigest() \ No newline at end of file + return grading_instruction_result \ No newline at end of file From 8020d92e5f7cac03d38883e63c6fe45292ef493c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20S=C3=B6lch?= Date: Tue, 28 Jan 2025 14:21:42 +0100 Subject: [PATCH 13/13] Remove whitespace from grading_criterion.py --- athena/athena/schemas/grading_criterion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/athena/athena/schemas/grading_criterion.py b/athena/athena/schemas/grading_criterion.py index 6d0d2f752..f13e23768 100644 --- a/athena/athena/schemas/grading_criterion.py +++ b/athena/athena/schemas/grading_criterion.py @@ -24,6 +24,6 @@ class GradingCriterion(Schema, ABC): structured_grading_instructions: List[StructuredGradingInstruction] = Field( [], example=[{"credits": 1.0, "gradingScale": "Good", "instructionDescription": "Some instructions", "feedback": "Nicely done!", "usageCount": 1}, {"credits": 0.0, "gradingScale": "Bad", "instructionDescription": "Some instructions", "feedback": "Try again!", "usageCount": 0}]) - + class StructuredGradingCriterion(BaseModel): - criteria: List[GradingCriterion] \ No newline at end of file + criteria: List[GradingCriterion]