prepare live demo

modules/programming/module_programming_llm/module_programming_llm/approaches/basic_by_file_approach.py

@@ -2,6 +2,7 @@
 
 from athena.programming import Submission, Exercise, Feedback
 from module_programming_llm.config import Configuration
+from module_programming_llm.helpers.web_search import bulk_search
 from module_programming_llm.helpers.models import ModelConfigType
 from module_programming_llm.prompts import GenerateFileSummary, SplitProblemStatementByFile, \
     SplitGradingInstructionsByFile, GenerateSuggestionsByFile, GenerateSuggestionsByFileOutput
@@ -13,6 +14,7 @@
     GenerateGradingCriterion, GenerateGradingCriterionOutput, GenerateGradingCriterionInput
 from module_programming_llm.prompts.generate_suggestions_by_file.generate_suggestions_by_file_input import \
     GenerateSuggestionsByFileInput
+from module_programming_llm.prompts.rag import RAGInput, RAG, RAGOutput
 from module_programming_llm.prompts.split_grading_instructions_by_file import SplitGradingInstructionsByFileOutput, \
     SplitGradingInstructionsByFileInput
 from module_programming_llm.prompts.split_problem_statement_by_file import SplitProblemStatementByFileOutput, \
@@ -62,6 +64,11 @@ async def generate_grading_criterion(step: GenerateGradingCriterion,
                                      model: ModelConfigType) -> Optional[GenerateGradingCriterionOutput]:  # type: ignore
     return await step.process(input_data, debug, model)
 
+async def generate_rag_queries(step: RAG,
+                                     input_data: RAGInput, debug: bool,
+                                     model: ModelConfigType) -> Optional[RAGOutput]:  # type: ignore
+    return await step.process(input_data, debug, model)
+
 
 async def generate_feedback(exercise: Exercise, submission: Submission, is_graded: bool,
                             module_config: Configuration) -> List[Feedback]:  # type: ignore
@@ -71,6 +78,13 @@ async def generate_feedback(exercise: Exercise, submission: Submission, is_grade
     is_debug = module_config.debug
     model = module_config.basic_by_file_approach.model
 
+#    rag_query_input = RAGInput(template_repo, solution_repo, exercise.id, exercise.problem_statement)
+#    rag_query_output = await generate_rag_queries(module_config.basic_by_file_approach.rag_requests, rag_query_input, module_config.debug, model)
+
+#    rag_result = "" if rag_query_output is None else bulk_search(rag_query_output.rag_queries, model)
+
+    rag_result = ""
+
     generate_file_summary_input = GenerateFileSummaryInput(template_repo, submission_repo, exercise.id, submission.id)
     file_summary_output = await generate_file_summary(module_config.basic_by_file_approach.generate_file_summary,
                                                       generate_file_summary_input, is_debug, model)
@@ -109,6 +123,7 @@ async def generate_feedback(exercise: Exercise, submission: Submission, is_grade
                                                                 submission.id, exercise.max_points,
                                                                 exercise.bonus_points, exercise.programming_language,
                                                                 file_summary_output.describe_solution_summary() if file_summary_output else "",
+                                                                rag_result,
                                                                 split_grading_instructions_output,
                                                                 split_problem_statement_output,
                                                                 exercise.grading_criteria, exercise.problem_statement,
@@ -123,16 +138,17 @@ async def generate_feedback(exercise: Exercise, submission: Submission, is_grade
         output = await filter_out_solutions(module_config.basic_by_file_approach.filter_out_solution,
                                             filter_out_solution_input, is_debug, model)
 
-    validate_suggestions_input = ValidateSuggestionsInput(solution_repo, template_repo, submission_repo,
-                                                          split_problem_statement_output, exercise.problem_statement,
-                                                          exercise.id, submission.id, output,
-                                                          split_grading_instructions_output, exercise.grading_criteria,
-                                                          exercise.grading_instructions,
-                                                          file_summary_output.describe_solution_summary() if file_summary_output else "",
-                                                          exercise.max_points, exercise.bonus_points,
-                                                          exercise.programming_language)
-    output = await validate_suggestions(
-        module_config.basic_by_file_approach.validate_suggestions, validate_suggestions_input, is_debug, model)
+    # validate_suggestions_input = ValidateSuggestionsInput(solution_repo, template_repo, submission_repo,
+    #                                                       split_problem_statement_output, exercise.problem_statement,
+    #                                                       exercise.id, submission.id, output,
+    #                                                       split_grading_instructions_output, exercise.grading_criteria,
+    #                                                       exercise.grading_instructions,
+    #                                                       file_summary_output.describe_solution_summary() if file_summary_output else "",
+    #                                                       exercise.max_points, exercise.bonus_points,
+    #                                                       exercise.programming_language,
+    #                                                       rag_result)
+    # output = await validate_suggestions(
+    #     module_config.basic_by_file_approach.validate_suggestions, validate_suggestions_input, is_debug, model)
 
     grading_instruction_ids = set(
         grading_instruction.id

modules/programming/module_programming_llm/module_programming_llm/config.py

@@ -9,6 +9,7 @@
 from module_programming_llm.prompts.filter_out_solution.filter_out_solution import FilterOutSolution
 from module_programming_llm.prompts.generate_grading_criterion.generate_grading_criterion import \
     GenerateGradingCriterion
+from module_programming_llm.prompts.rag import RAG
 from module_programming_llm.prompts.validate_suggestions import ValidateSuggestions
 
 
@@ -39,6 +40,7 @@ class BasicByFileApproachConfig(BasicApproachConfig, ABC):
                                      description="Maximum number of files. If exceeded, it will prioritize the most important ones.")
     tokens_before_split: int = Field(default=250,
                                      description="Split the grading instructions into file-based ones after this number of tokens.")
+    rag_requests: RAG = Field(default=RAG())
 
 
 # class ZeroShotApproachConfig(BasicApproachConfig, ABC):

modules/programming/module_programming_llm/module_programming_llm/helpers/models/openai.py

@@ -6,7 +6,7 @@
 from enum import Enum
 from pydantic import Field, field_validator, PositiveInt
 from langchain.base_language import BaseLanguageModel
-from langchain_community.chat_models import ChatOpenAI
+from langchain_openai.chat_models import ChatOpenAI
 from langchain_openai import AzureChatOpenAI
 
 from athena.logger import logger

modules/programming/module_programming_llm/module_programming_llm/helpers/web_search.py

@@ -0,0 +1,86 @@
+from typing import Sequence, List
+
+from langchain.chains.qa_with_sources.retrieval import RetrievalQAWithSourcesChain
+from langchain_community.document_loaders import AsyncChromiumLoader
+from langchain_community.document_transformers import BeautifulSoupTransformer
+from duckduckgo_search import DDGS
+import re
+
+from langchain_community.retrievers import WebResearchRetriever
+from langchain_community.utilities import GoogleSearchAPIWrapper
+from langchain_community.vectorstores import Chroma
+from langchain_core.documents import Document
+from langchain_core.tools import Tool
+from langchain_openai import OpenAIEmbeddings
+
+from module_programming_llm.helpers.models import ModelConfigType
+
+
+def bulk_search(queries: Sequence[str], model: ModelConfigType) -> List[str]:
+    result = []
+    for query in queries:
+        result.append(answer_query(query, model))
+    return result
+#
+#
+# def search(query: str) -> List[str]:
+#     results = DDGS().text(query, max_results=5)
+#     urls = []
+#     for result in results:
+#         url = result['href']
+#         urls.append(url)
+#
+#     docs = get_page(urls)
+#
+#     content = []
+#     for doc in docs:
+#         page_text = re.sub("\n\n+", "\n", doc.page_content)
+#         text = truncate(page_text)
+#         content.append(text)
+#
+#     return content
+#
+#
+# def get_page(urls: List[str]) -> Sequence[Document]:
+#     loader = AsyncChromiumLoader(urls, headless=True)
+#     html = loader.load()
+#
+#     bs_transformer = BeautifulSoupTransformer()
+#     docs_transformed = bs_transformer.transform_documents(html, tags_to_extract=["p"], remove_unwanted_tags=["a"])
+#
+#     return docs_transformed
+#
+#
+# def truncate(text) -> str:
+#     words = text.split()
+#     truncated = " ".join(words[:1000])
+#
+#     return truncated
+
+def answer_query(query, model: ModelConfigType):
+    model = model.get_model()  # type: ignore[attr-defined]
+    vectorstore = Chroma(
+        embedding_function=OpenAIEmbeddings(), persist_directory="./chroma_db_oai"
+    )
+
+    # Search
+    search = GoogleSearchAPIWrapper()
+
+    # # Initialize
+    web_search_retriever = WebResearchRetriever.from_llm(
+        vectorstore=vectorstore, llm=model, search=search, allow_dangerous_requests=True
+    )
+    qa_chain = RetrievalQAWithSourcesChain.from_chain_type(
+        model, retriever=web_search_retriever
+    )
+    result = qa_chain({"question": query})
+
+    search = GoogleSearchAPIWrapper()
+
+    tool = Tool(
+        name="google_search",
+        description="Search Google for recent results.",
+        func=search.run,
+    )
+
+    return tool.run(query)

modules/programming/module_programming_llm/module_programming_llm/prompts/generate_grading_criterion/generate_grading_criterion.py

@@ -42,10 +42,21 @@ async def process(self, input_data: GenerateGradingCriterionInput, debug: bool,
             name_only=True
         ).split("\n")
 
-        changed_files = load_files_from_repo(
-            input_data.solution_repo,
-            file_filter=lambda file_path: file_path in changed_files_from_template_to_solution,
+        all_changed_files = load_files_from_repo(
+            input_data.solution_repo
         )
+        changed_files = {}
+        changed_files_content = ""
+        for file in changed_files_from_template_to_solution:
+            if not file.endswith('.pbxproj'):
+                changed_files[file] = get_diff(
+                    src_repo=input_data.template_repo,
+                    dst_repo=input_data.solution_repo,
+                    src_prefix="template",
+                    dst_prefix="solution",
+                    file_path=file,
+                )
+                changed_files_content += "\n" + file + ":" + changed_files[file]
 
         prompt = get_chat_prompt_with_formatting_instructions(
             model=model,
@@ -56,7 +67,6 @@ async def process(self, input_data: GenerateGradingCriterionInput, debug: bool,
 
         prompt_input = {
             "problem_statement": input_data.problem_statement or "No problem statement.",
-            "changed_files_from_template_to_solution": ", ".join(changed_files_from_template_to_solution),
             "grading_instructions": input_data.grading_instructions,
             "max_points": input_data.max_points,
             "bonus_points": input_data.bonus_points,

modules/programming/module_programming_llm/module_programming_llm/prompts/generate_suggestions_by_file/__init__.py

@@ -1,4 +1,5 @@
 from .generate_suggestions_by_file import GenerateSuggestionsByFile
+from .generate_suggestions_by_file_input import GenerateSuggestionsByFileInput
 from .generate_suggestions_by_file_output import GenerateSuggestionsByFileOutput
 
-__all__ = ['GenerateSuggestionsByFile', 'GenerateSuggestionsByFileOutput']
+__all__ = ['GenerateSuggestionsByFile', 'GenerateSuggestionsByFileOutput', 'GenerateSuggestionsByFileInput']

modules/programming/module_programming_llm/module_programming_llm/prompts/generate_suggestions_by_file/generate_suggestions_by_file.py

@@ -153,12 +153,12 @@ async def process(self, input_data: GenerateSuggestionsByFileInput, debug: bool,
                     "submission_file": file_content,
                     "max_points": input_data.max_points,
                     "bonus_points": input_data.bonus_points,
-                    "solution_to_submission_diff": solution_to_submission_diff,
                     "template_to_submission_diff": template_to_submission_diff,
                     "template_to_solution_diff": template_to_solution_diff,
                     "grading_instructions": grading_instructions,
                     "problem_statement": problem_statement,
-                    "solution_summary": input_data.solution_summary
+                    "solution_summary": input_data.solution_summary,
+                    "rag_data": input_data.rag_data
                 }
             )
 

modules/programming/module_programming_llm/module_programming_llm/prompts/generate_suggestions_by_file/generate_suggestions_by_file_input.py

@@ -25,6 +25,7 @@ class GenerateSuggestionsByFileInput:
     bonus_points: float
     programming_language: str
     solution_summary: str
+    rag_data: List[str]
 
     def __init__(self,
                  template_repo: Repo,
@@ -36,6 +37,7 @@ def __init__(self,
                  bonus_points: float,
                  programming_language: str,
                  solution_summary: str,
+                 rag_data: List[str],
                  grading_instructions_by_file: Optional[SplitGradingInstructionsByFileOutput] = None,
                  problem_statement_by_file: Optional[SplitProblemStatementByFileOutput] = None,
                  grading_criteria: Optional[List[GradingCriterion]] = None,
@@ -55,3 +57,4 @@ def __init__(self,
         self.bonus_points = bonus_points
         self.programming_language = programming_language
         self.solution_summary = solution_summary
+        self.rag_data = rag_data

modules/programming/module_programming_llm/module_programming_llm/prompts/generate_suggestions_by_file/prompt.py

@@ -3,7 +3,8 @@
 
 # Task
 Create graded feedback suggestions for a student\'s programming submission that a human tutor would accept. \
-Meaning, the feedback you provide should be appliable to the submission with little to no modification.
+Meaning, the feedback you provide should be applicable to the submission with little to no modification.
+Give points for correct answers. Subtract points for wrong answers. Give 0 points for neutral answers.
 
 # Style
 1. Constructive, 2. Specific, 3. Balanced, 4. Clear and Concise, 5. Actionable, 6. Educational, 7. Contextual
@@ -15,11 +16,14 @@
 {grading_instructions}
 Max points: {max_points}, bonus points: {bonus_points} (whole assessment, not just this file)
 
-# Diff between solution (deletions) and student\'s submission (additions):
-{solution_to_submission_diff}
+# Diff between template (deletions) and solution (additions):
+{template_to_solution_diff}
 
 # Summary of other solution files
 {solution_summary}
+
+# RAG data
+{rag_data}
 """
 
 human_message = """\

modules/programming/module_programming_llm/module_programming_llm/prompts/pipeline_step.py

@@ -10,7 +10,7 @@
 
 
 class PipelineStep(BaseModel, Generic[TInput, TOutput]):
-    max_input_tokens: int = Field(default=3000, description="Maximum number of tokens in the input prompt.")
+    max_input_tokens: int = Field(default=10000, description="Maximum number of tokens in the input prompt.")
 
     @abstractmethod
     async def process(self, input_data: TInput, debug: bool, model: ModelConfigType) -> Optional[TOutput]: # type: ignore

modules/programming/module_programming_llm/module_programming_llm/prompts/rag/__init__.py

@@ -0,0 +1,5 @@
+from .rag import RAG
+from .rag_input import RAGInput
+from .rag_output import RAGOutput
+
+__all__ = ['RAG', 'RAGInput', 'RAGOutput']

modules/programming/module_programming_llm/module_programming_llm/prompts/rag/prompt.py

@@ -0,0 +1,23 @@
+system_message = """\
+You are an AI tutor for programming assessment at a prestigious university.
+
+# Task
+Identify, if you understand the problem and surrounding information completely.
+In case you do not understand something, formulate up to 2 specific questions that will help you understand the problem statement better.
+
+# Style
+1. Constructive, 2. Specific, 3. Balanced, 4. Clear and Concise, 5. Contextual
+
+For testing purposes, assume you do not know anything about sorting
+"""
+
+human_message = """\
+# Problem statement
+{problem_statement}
+
+Changed files from template to sample solution:
+{changed_files_from_template_to_solution}
+
+# Diff between template (deletions) and sample solution(additions):
+{template_to_solution_diff}
+"""