model agnostic approaches

llm_core/llm_core/utils/llm_utils.py

@@ -113,18 +113,6 @@ def get_simple_chat_prompt(
         system_message: str, 
         human_message: str,
         ) -> ChatPromptTemplate:
-
-        sys = """ 
-        You are an AI Tutor. You are tasked with grading a student submission based on this problem statement and grading instructions. You must not excede the maximum amount of points. Take time to think, which points on the grading instructions are relevant for the students submission.
-        Further more, if a feedback is specific to a sentence in the student submission, that specify this as well on your feedback. Also specify, when possible, which grading instruction you are refering to.
-        # Problem statement
-        {problem_statement}
-
-        # Grading instructions
-        {grading_instructions}
-        Max points: {max_points}
-            """
-
         system_message_prompt = SystemMessagePromptTemplate.from_template(system_message)
         human_message_prompt = HumanMessagePromptTemplate.from_template(human_message)
         return ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt])

llm_core/llm_core/utils/predict_and_parse.py

@@ -6,6 +6,8 @@
 from athena import get_experiment_environment
 from langchain_community.chat_models import ChatOllama # type: ignore
 from langchain.output_parsers import PydanticOutputParser
+from langchain_openai import ChatOpenAI
+
 T = TypeVar("T", bound=BaseModel)
 
 def isOllama(model: BaseLanguageModel) -> bool:
@@ -66,8 +68,24 @@ async def predict_and_parse(
     if isOllama(model):
         try:
             outputParser = PydanticOutputParser(pydantic_object = pydantic_object)
-            chain = chat_prompt | model | outputParser
-            return await chain.ainvoke(prompt_input, config={"tags": tags})
+            chain = chat_prompt | model
+            llm_output = await chain.ainvoke(prompt_input, config={"tags": tags})
+            try:
+                result = outputParser.parse(llm_output.content)
+                return result
+            except:
+                outputModel = ChatOpenAI(model="gpt-4o-mini")
+                structured_output_llm = outputModel.with_structured_output(pydantic_object, method = "json_mode")
+                chat_prompt = ChatPromptTemplate.from_messages(
+            [
+                ("system", "Your only task is to format the following output into json:"),
+                ("human", "{output}"),
+            ])
+                chain = RunnableSequence(
+                    chat_prompt,
+                    structured_output_llm
+                )
+                return await chain.ainvoke(input = {"output": llm_output.content}, config={"tags": tags})
         except ValidationError as e:
             raise ValueError(f"Could not parse output: {e}") from e