From 672f76210bbe177776cac4ff13a8be315384b4bf Mon Sep 17 00:00:00 2001 From: leon Date: Sat, 7 Dec 2024 12:39:56 +0100 Subject: [PATCH] Refactor grading criterion handling: remove structured grading criterion model, update imports --- athena/athena/app.py | 2 +- athena/athena/models/db_exercise.py | 2 -- athena/athena/schemas/__init__.py | 3 +- athena/athena/schemas/grading_criterion.py | 5 ++- .../schemas/structured_grading_criterion.py | 7 ---- .../structured_grading_criterion_storage.py | 31 ----------------- .../core/generate_suggestions.py | 2 +- .../get_structured_grading_instructions.py | 34 ++----------------- 8 files changed, 10 insertions(+), 76 deletions(-) delete mode 100644 athena/athena/schemas/structured_grading_criterion.py delete mode 100644 athena/athena/storage/structured_grading_criterion_storage.py diff --git a/athena/athena/app.py b/athena/athena/app.py index e9d653f03..b73cc045d 100644 --- a/athena/athena/app.py +++ b/athena/athena/app.py @@ -69,4 +69,4 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE return JSONResponse( status_code=422, content={"detail": exc.errors()}, - ) \ No newline at end of file + ) diff --git a/athena/athena/models/db_exercise.py b/athena/athena/models/db_exercise.py index d0c5e4909..323cc5527 100644 --- a/athena/athena/models/db_exercise.py +++ b/athena/athena/models/db_exercise.py @@ -18,8 +18,6 @@ class DBExercise(Model, Base): grading_criteria = Column(JSON, nullable=True) meta = Column(JSON, nullable=False) - structured_grading_criterion = relationship("DBStructuredGradingCriterion", back_populates="exercise", uselist=False) # Use uselist=False for one-to-one - # Polymorphism, discriminator attribute type = Column(SqlEnum(ExerciseType), index=True, nullable=False) diff --git a/athena/athena/schemas/__init__.py b/athena/athena/schemas/__init__.py index b5f6a7943..244c2d9c0 100644 --- a/athena/athena/schemas/__init__.py +++ b/athena/athena/schemas/__init__.py @@ -13,5 +13,4 @@ from .modeling_feedback import ModelingFeedback from .modeling_exercise import ModelingExercise from .modeling_submission import ModelingSubmission -from .grading_criterion import GradingCriterion, StructuredGradingInstruction -from .structured_grading_criterion import StructuredGradingCriterion \ No newline at end of file +from .grading_criterion import GradingCriterion, StructuredGradingInstruction, StructuredGradingCriterion \ No newline at end of file diff --git a/athena/athena/schemas/grading_criterion.py b/athena/athena/schemas/grading_criterion.py index 186478830..6d0d2f752 100644 --- a/athena/athena/schemas/grading_criterion.py +++ b/athena/athena/schemas/grading_criterion.py @@ -23,4 +23,7 @@ class GradingCriterion(Schema, ABC): title: Optional[str] = Field(None, example="Some instructions") structured_grading_instructions: List[StructuredGradingInstruction] = Field( [], example=[{"credits": 1.0, "gradingScale": "Good", "instructionDescription": "Some instructions", "feedback": "Nicely done!", "usageCount": 1}, - {"credits": 0.0, "gradingScale": "Bad", "instructionDescription": "Some instructions", "feedback": "Try again!", "usageCount": 0}]) \ No newline at end of file + {"credits": 0.0, "gradingScale": "Bad", "instructionDescription": "Some instructions", "feedback": "Try again!", "usageCount": 0}]) + +class StructuredGradingCriterion(BaseModel): + criteria: List[GradingCriterion] \ No newline at end of file diff --git a/athena/athena/schemas/structured_grading_criterion.py b/athena/athena/schemas/structured_grading_criterion.py deleted file mode 100644 index 89f1beb7b..000000000 --- a/athena/athena/schemas/structured_grading_criterion.py +++ /dev/null @@ -1,7 +0,0 @@ -from typing import List -from athena.schemas.grading_criterion import GradingCriterion -from pydantic import BaseModel - - -class StructuredGradingCriterion(BaseModel): - criteria: List[GradingCriterion] \ No newline at end of file diff --git a/athena/athena/storage/structured_grading_criterion_storage.py b/athena/athena/storage/structured_grading_criterion_storage.py deleted file mode 100644 index 459e4c02d..000000000 --- a/athena/athena/storage/structured_grading_criterion_storage.py +++ /dev/null @@ -1,31 +0,0 @@ -from typing import Optional -from athena.contextvars import get_lms_url -from athena.database import get_db - -from athena.models import DBStructuredGradingCriterion -from athena.schemas import StructuredGradingCriterion - -def get_structured_grading_criterion(exercise_id: int, current_hash: Optional[str] = None) -> Optional[StructuredGradingCriterion]: - lms_url = get_lms_url() - with get_db() as db: - cache_entry = db.query(DBStructuredGradingCriterion).filter_by( - exercise_id=exercise_id, lms_url=lms_url - ).first() - if cache_entry is not None and (current_hash is None or cache_entry.instructions_hash == current_hash): # type: ignore - return StructuredGradingCriterion.parse_obj(cache_entry.structured_grading_criterion) - return None - -def store_structured_grading_criterion( - exercise_id: int, hash: str, structured_instructions: StructuredGradingCriterion -): - lms_url = get_lms_url() - with get_db() as db: - db.merge( - DBStructuredGradingCriterion( - exercise_id=exercise_id, - lms_url=lms_url, - instructions_hash=hash, - structured_grading_criterion=structured_instructions.dict(), - ) - ) - db.commit() \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py index 61f051348..6db69e7d2 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py @@ -1,4 +1,4 @@ -from athena.schemas.structured_grading_criterion import StructuredGradingCriterion +from athena.schemas.grading_criterion import StructuredGradingCriterion from langchain_core.output_parsers import PydanticOutputParser from langchain_core.prompts import ChatPromptTemplate diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py index 75b7ded47..ae84e0ddf 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py @@ -1,13 +1,9 @@ -import hashlib -import json -from typing import Any, Dict, List, Optional -from athena import logger +from typing import List, Optional from athena.metadata import emit_meta -from athena.storage.structured_grading_criterion_storage import get_structured_grading_criterion, store_structured_grading_criterion from langchain_core.output_parsers import PydanticOutputParser from langchain_core.prompts import ChatPromptTemplate -from athena.schemas import GradingCriterion, StructuredGradingCriterion +from athena.schemas.grading_criterion import GradingCriterion, StructuredGradingCriterion from llm_core.utils.predict_and_parse import predict_and_parse from module_modeling_llm.config import BasicApproachConfig from module_modeling_llm.models.exercise_model import ExerciseModel @@ -23,13 +19,6 @@ async def get_structured_grading_instructions( if grading_criteria: return StructuredGradingCriterion(criteria=grading_criteria) - - # Check if we have cached instructions for this exercise - current_hash = get_grading_instructions_hash(exercise_model) - cached_instructions = get_structured_grading_criterion(exercise_model.exercise_id, current_hash) - if cached_instructions: - print("Using cached instructions") - return cached_instructions chat_prompt = ChatPromptTemplate.from_messages([ ("system", config.generate_suggestions_prompt.structured_grading_instructions_system_message), @@ -64,22 +53,5 @@ async def get_structured_grading_instructions( if not grading_instruction_result: raise ValueError("No structured grading instructions were returned by the model.") - - # Cache the grading instructions - hash = get_grading_instructions_hash(exercise_model) - store_structured_grading_criterion(exercise_model.exercise_id, hash, grading_instruction_result) - - return grading_instruction_result - -def get_grading_instructions_hash(exercise: ExerciseModel) -> str: - - hashable_data = { - "problem_statement": exercise.problem_statement, - "grading_instructions": exercise.grading_instructions, - "sample_solution": exercise.transformed_example_solution, - "max_points": exercise.max_points, - "bonus_points": exercise.bonus_points, - } - json_string = json.dumps(hashable_data, sort_keys=True, default=str) - return hashlib.sha256(json_string.encode()).hexdigest() \ No newline at end of file + return grading_instruction_result \ No newline at end of file