Upgrade Playground

assessment_module_manager/Dockerfile

@@ -26,10 +26,12 @@ RUN poetry config virtualenvs.create true \
 COPY . ./
 
 # Use modules configuration file for docker
-RUN mv ./modules.docker.ini ./modules.ini
+RUN cp ./modules.docker.ini ./modules.ini
+RUN rm ./modules.docker.ini
 
 # Use deployments configuration file for docker
-RUN mv ./deployments.docker.ini ./deployments.ini
+RUN cp ./deployments.docker.ini ./deployments.ini
+RUN rm ./deployments.docker.ini
 
 # poetry scripts don't work here
-CMD poetry run python -m assessment_module_manager
+CMD poetry run python -m assessment_module_manager

assessment_module_manager/assessment_module_manager/endpoints/health_endpoint.py

@@ -37,7 +37,9 @@ class HealthResponse(BaseModel):
                     "url": "http://localhost:5001",
                     "type": "programming",
                     "healthy": True,
-                    "supportsEvaluation": True
+                    "supportsEvaluation": True,
+                    "supportsNonGradedFeedbackRequests": True,
+                    "supportsGradedFeedbackRequests": True
                 }
             }
         ]
@@ -58,7 +60,9 @@ async def get_health() -> HealthResponse:
                 "url": module.url,
                 "type": module.type,
                 "healthy": await is_healthy(module),
-                "supportsEvaluation": module.supports_evaluation
+                "supportsEvaluation": module.supports_evaluation,
+                "supportsNonGradedFeedbackRequests": module.supports_non_graded_feedback_requests,
+                "supportsGradedFeedbackRequests": module.supports_graded_feedback_requests
             }
             for module in get_modules()
         }

assessment_module_manager/assessment_module_manager/module/list_modules.py

@@ -21,6 +21,8 @@ def list_modules() -> List[Module]:
             url=cast(AnyHttpUrl, os.environ.get(f"{module.upper()}_URL", modules_config[module]["url"])),
             type=ExerciseType(modules_config[module]["type"]),
             supports_evaluation=modules_config[module].getboolean("supports_evaluation"),
+            supports_non_graded_feedback_requests=modules_config[module].getboolean("supports_non_graded_feedback_requests"),
+            supports_graded_feedback_requests=modules_config[module].getboolean("supports_graded_feedback_requests")
         )
         for module in modules_config.sections()
     ]

assessment_module_manager/assessment_module_manager/module/module.py

@@ -9,3 +9,5 @@ class Module(BaseModel):
     url: AnyHttpUrl = Field(example="http://localhost:5001")
     type: ExerciseType = Field(example=ExerciseType.text)
     supports_evaluation: bool = Field(description="Whether the module supports evaluation", example=True)
+    supports_non_graded_feedback_requests: bool = Field(description="Whether the module supports non-graded feedback requests", example=True),
+    supports_graded_feedback_requests: bool = Field(description="Whether the module supports graded feedback requests", example=True)

assessment_module_manager/deployments.docker.ini

@@ -1,2 +1,8 @@
 [schwind]
-url = https://ma-schwind.ase.cit.tum.de
+url = https://ma-schwind.ase.cit.tum.de
+
+[playground]
+url = https://athenetest1-03.ase.cit.tum.de
+
+[localhost]
+url = http://localhost:3000

assessment_module_manager/deployments.ini

@@ -1,4 +1,7 @@
 # for local development only
 
 [local]
-url = http://localhost:8080
+url = http://localhost:8080
+
+[playground]
+url = http://localhost:3000

assessment_module_manager/modules.docker.ini

@@ -2,33 +2,47 @@
 url = http://module-example:5001
 type = programming
 supports_evaluation = true
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = false
 
 [module_programming_llm]
 url = http://module-programming-llm:5002
 type = programming
 supports_evaluation = false
+supports_non_graded_feedback_requests = true
+supports_graded_feedback_requests = true
 
 [module_text_llm]
 url = http://module-text-llm:5003
 type = text
 supports_evaluation = true
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = true
 
 [module_text_cofee]
 url = http://module-text-cofee:5004
 type = text
 supports_evaluation = false
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = true
 
 [module_programming_themisml]
 url = http://module-programming-themisml:5005
 type = programming
 supports_evaluation = false
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = true
 
 [module_programming_apted]
 url = http://module-programming-apted:5006
 type = programming
 supports_evaluation = false
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = false
 
 [module_modeling_llm]
 url = http://module-modeling-llm:5008
 type = modeling
 supports_evaluation = false
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = true

assessment_module_manager/modules.ini

@@ -2,33 +2,47 @@
 url = http://localhost:5001
 type = programming
 supports_evaluation = true
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = false
 
 [module_programming_llm]
 url = http://localhost:5002
 type = programming
 supports_evaluation = false
+supports_non_graded_feedback_requests = true
+supports_graded_feedback_requests = true
 
 [module_text_llm]
 url = http://localhost:5003
 type = text
 supports_evaluation = true
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = true
 
 [module_text_cofee]
 url = http://localhost:5004
 type = text
 supports_evaluation = false
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = true
 
 [module_programming_themisml]
 url = http://localhost:5005
 type = programming
 supports_evaluation = false
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = true
 
 [module_programming_apted]
 url = http://localhost:5006
 type = programming
 supports_evaluation = false
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = false
 
 [module_modeling_llm]
 url = http://localhost:5008
 type = modeling
 supports_evaluation = false
+supports_non_graded_feedback_requests = false
+supports_graded_feedback_requests = true

athena/athena/schemas/feedback.py

@@ -29,8 +29,8 @@ class Feedback(Schema, ABC):
     exercise_id: int = Field(example=1)
     submission_id: int = Field(example=1)
 
-    def to_model(self, is_suggestion: bool = False, lms_id: Optional[int] = None):
-        return type(self).get_model_class()(**self.dict(), is_suggestion=is_suggestion, lms_id=lms_id)
+    def to_model(self, is_suggestion: bool = False, lms_id: Optional[int] = None, lms_url: Optional[str] = None):
+        return type(self).get_model_class()(**self.dict(), is_suggestion=is_suggestion, lms_id=lms_id, lms_url=lms_url)
 
     class Config:
         orm_mode = True

athena/athena/storage/feedback_storage.py

@@ -59,7 +59,7 @@ def store_feedback(feedback: Feedback, is_lms_id=False, lms_url: Optional[str] =
                                                                  lms_url=lms_url).scalar()
             feedback.id = internal_id
 
-        stored_feedback_model = db.merge(feedback.to_model(lms_id=lms_id))
+        stored_feedback_model = db.merge(feedback.to_model(lms_id=lms_id, lms_url=lms_url))
         db.commit()
         return stored_feedback_model.to_schema()
 

env_example/assessment_module_manager.env

@@ -12,4 +12,4 @@ MODULE_PROGRAMMING_THEMISML_SECRET=12345abcdef
 # LMS Deployments                                          #
 ################################################################
 # the deployment name should correspond to the name in deployments.ini
-LMS_DEPLOYMENT_NAME_SECRET=12345abcdef
+LMS_DEPLOYMENT_NAME_SECRET=12345abcdef

module_modeling_llm/module_modeling_llm/__main__.py

@@ -28,7 +28,7 @@ def process_incoming_feedback(exercise: Exercise, submission: Submission, feedba
 
 
 @feedback_provider
-async def suggest_feedback(exercise: Exercise, submission: Submission, module_config: Configuration) -> List[Feedback]:
+async def suggest_feedback(exercise: Exercise, submission: Submission, is_graded: bool, module_config: Configuration) -> List[Feedback]:
     logger.info("suggest_feedback: Suggestions for submission %d of exercise %d were requested", submission.id,
                 exercise.id)
     return await generate_suggestions(exercise, submission, module_config.approach, module_config.debug)

module_programming_llm/module_programming_llm/generate_summary_by_file.py

@@ -91,7 +91,7 @@ async def generate_summary_by_file(
         model=model,
         system_message=config.generate_file_summary_prompt.system_message,
         human_message=config.generate_file_summary_prompt.human_message,
-        pydantic_object=FileDescription,
+        pydantic_object=SolutionSummary,
     )
 
     prompt_inputs = []

module_programming_llm/module_programming_llm/split_problem_statement_by_file.py

@@ -80,7 +80,7 @@ async def split_problem_statement_by_file(
         "changed_files_from_template_to_submission": ", ".join(changed_files_from_template_to_submission)
     }
 
-    if "changed_files_from_template_to_solution" in prompt.input_variables:
+    if "changed_files_from_template_to_solution" in chat_prompt.input_variables:
         solution_repo = exercise.get_solution_repository()
         changed_files_from_template_to_solution = get_diff(
             src_repo=template_repo,

module_programming_themisml/module_programming_themisml/__main__.py

@@ -81,7 +81,7 @@ def process_incoming_feedback(exercise: Exercise, submission: Submission, feedba
 
 
 @feedback_provider
-async def suggest_feedback(exercise: Exercise, submission: Submission) -> List[Feedback]:
+async def suggest_feedback(exercise: Exercise, submission: Submission, is_graded: bool = True) -> List[Feedback]:
     logger.info("suggest_feedback: Suggestions for submission %d of exercise %d were requested", submission.id, exercise.id)
 
     # ThemisML currently only works with Java

module_text_cofee/module_text_cofee/__main__.py

@@ -45,7 +45,7 @@ def process_incoming_feedback(exercise: Exercise, submission: Submission, feedba
 
 
 @feedback_provider
-def suggest_feedback(exercise: Exercise, submission: Submission) -> List[Feedback]:
+def suggest_feedback(exercise: Exercise, submission: Submission, is_graded: bool = True) -> List[Feedback]:
     logger.info(
         "suggest_feedback: Suggestions for submission %d of exercise %d were requested",
         submission.id, exercise.id

module_text_llm/module_text_llm/__main__.py

@@ -32,8 +32,9 @@ def process_incoming_feedback(exercise: Exercise, submission: Submission, feedba
 
 
 @feedback_provider
-async def suggest_feedback(exercise: Exercise, submission: Submission, module_config: Configuration) -> List[Feedback]:
-    logger.info("suggest_feedback: Suggestions for submission %d of exercise %d were requested", submission.id, exercise.id)
+async def suggest_feedback(exercise: Exercise, submission: Submission, is_graded: bool, module_config: Configuration) -> List[Feedback]:
+    logger.info("suggest_feedback: %s suggestions for submission %d of exercise %d were requested",
+                "Graded" if is_graded else "Non-graded", submission.id, exercise.id)
     return await generate_suggestions(exercise, submission, module_config.approach, module_config.debug)
 
 

playground/Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM node:18.16.0-alpine
+FROM node:18.17.0-alpine
 
 LABEL org.opencontainers.image.source=https://github.com/ls1intum/Athena
 

playground/data/example/exercise-1.json

@@ -8,15 +8,15 @@
   "problem_statement": "-> file:problem-statement.md",
   "grading_instructions": "-> file:grading-instructions.md",
   "programming_language": "java",
-  "solution_repository_url": "{{exerciseDataUrl}}/solution.zip",
-  "template_repository_url": "{{exerciseDataUrl}}/template.zip",
-  "tests_repository_url": "{{exerciseDataUrl}}/tests.zip",
+  "solution_repository_uri": "{{exerciseDataUrl}}/solution.zip",
+  "template_repository_uri": "{{exerciseDataUrl}}/template.zip",
+  "tests_repository_uri": "{{exerciseDataUrl}}/tests.zip",
   "meta": {},
 
   "submissions": [
     {
       "id": 101,
-      "repository_url": "{{exerciseDataUrl}}/submissions/101.zip",
+      "repository_uri": "{{exerciseDataUrl}}/submissions/101.zip",
       "meta": {},
       "feedbacks": [
         {
@@ -43,12 +43,12 @@
     },
     {
       "id": 102,
-      "repository_url": "{{exerciseDataUrl}}/submissions/102.zip",
+      "repository_uri": "{{exerciseDataUrl}}/submissions/102.zip",
       "meta": {}
     },
     {
       "id": 103,
-      "repository_url": "{{exerciseDataUrl}}/submissions/103.zip",
+      "repository_uri": "{{exerciseDataUrl}}/submissions/103.zip",
       "meta": {},
       "feedbacks": [
         {