Merge pull request #130 from Yiannis128/dev

Add tests for Solution Generator and children

Pipfile

@@ -32,7 +32,6 @@ pytest = "*"
 pytest-cov = "*"
 pytest-regtest = "*"
 py = "*" # Dependency of pytest-regtest
-build = "*"
 twine = "*"
 hatch = "*"
 transformers = "*"

esbmc_ai/__about__.py

@@ -1,4 +1,4 @@
 # Author: Yiannis Charalambous
 
-__version__ = "v0.5.0rc2"
+__version__ = "v0.5.0rc4"
 __author__: str = "Yiannis Charalambous"

esbmc_ai/reverse_order_solution_generator.py

@@ -1,15 +1,9 @@
 # Author: Yiannis Charalambous
 
-from langchain.schema import BaseMessage, HumanMessage
+from langchain.schema import BaseMessage
 from typing_extensions import override, Optional
-from esbmc_ai.solution_generator import (
-    SolutionGenerator,
-    get_source_code_formatted,
-    get_source_code_err_line_idx,
-    get_clang_err_line_index,
-    apply_line_patch,
-)
-from esbmc_ai.chat_response import FinishReason, ChatResponse
+from esbmc_ai.solution_generator import SolutionGenerator
+from esbmc_ai.chat_response import ChatResponse
 
 # TODO Test me
 

tests/test_latest_state_solution_generator.py

@@ -1,11 +1,13 @@
 # Author: Yiannis Charalambous
 
+from typing import Optional
 import pytest
 
 from langchain.schema import HumanMessage, AIMessage, SystemMessage
 from langchain_community.llms.fake import FakeListLLM
 
 from esbmc_ai.ai_models import AIModel
+from esbmc_ai.chat_response import ChatResponse
 from esbmc_ai.config import AIAgentConversation, ChatPromptSettings
 from esbmc_ai.latest_state_solution_generator import LatestStateSolutionGenerator
 
@@ -23,9 +25,8 @@ def setup_llm_model():
     return llm, model
 
 
-def test_call_update_state_first(setup_llm_model) -> None:
+def test_send_message(setup_llm_model) -> None:
     llm, model = setup_llm_model
-
     chat_settings = ChatPromptSettings(
         system_messages=AIAgentConversation(
             messages=(
@@ -44,8 +45,25 @@ def test_call_update_state_first(setup_llm_model) -> None:
         ai_model_agent=chat_settings,
     )
 
-    with pytest.raises(AssertionError):
-        solution_generator.generate_solution()
+    def send_message_mock(message: Optional[str] = None) -> ChatResponse:
+        assert len(solution_generator.messages) == 1
+        assert solution_generator.messages[0] == HumanMessage(
+            content=chat_settings.initial_prompt,
+        )
+
+        return ChatResponse()
+
+    # Use the LLM method to check if the code is overwritten
+    solution_generator.send_message = send_message_mock
+
+    solution_generator.update_state("", "")
+
+    solution_generator.generate_solution()
+    chat_settings.initial_prompt = "aaaaaaa"
+    solution_generator.generate_solution()
+    chat_settings.initial_prompt = "bbbbbbb"
+    solution_generator.generate_solution()
+    chat_settings.initial_prompt = "ccccccc"
 
 
 def test_message_stack(setup_llm_model) -> None:

tests/test_reverse_order_solution_generator.py

@@ -2,7 +2,11 @@
 
 import pytest
 
-from langchain.schema import HumanMessage, AIMessage, SystemMessage
+from langchain.schema import (
+    HumanMessage,
+    AIMessage,
+    SystemMessage,
+)
 from langchain_community.llms.fake import FakeListLLM
 
 from esbmc_ai.ai_models import AIModel
@@ -23,29 +27,10 @@ def setup_llm_model():
     return llm, model
 
 
-def test_call_update_state_first(setup_llm_model) -> None:
-    llm, model = setup_llm_model
-
-    chat_settings = ChatPromptSettings(
-        system_messages=AIAgentConversation(
-            messages=(
-                SystemMessage(content="Test message 1"),
-                HumanMessage(content="Test message 2"),
-                AIMessage(content="Test message 3"),
-            ),
-        ),
-        initial_prompt="Initial test message",
-        temperature=1.0,
-    )
-
-    solution_generator = ReverseOrderSolutionGenerator(
-        llm=llm,
-        ai_model=model,
-        ai_model_agent=chat_settings,
-    )
-
-    with pytest.raises(AssertionError):
-        solution_generator.generate_solution()
+def test_send_message(setup_llm_model) -> None:
+    # TODO Test the send_message method to ensure it actually reverses the
+    # messages as intended.
+    pass
 
 
 def test_message_stack(setup_llm_model) -> None:

tests/test_solution_generator.py

@@ -1,8 +1,52 @@
 # Author: Yiannis Charalambous
 
+from langchain.schema import AIMessage, HumanMessage, SystemMessage
+from langchain_core.language_models import FakeListLLM
+import pytest
+
+from esbmc_ai.ai_models import AIModel
+from esbmc_ai.config import AIAgentConversation, ChatPromptSettings
 from esbmc_ai.solution_generator import SolutionGenerator
 
 
+@pytest.fixture(scope="function")
+def setup_llm_model():
+    llm = FakeListLLM(
+        responses=[
+            "This is a test response",
+            "Another test response",
+            "One more!",
+        ],
+    )
+    model = AIModel("test model", 1000)
+    return llm, model
+
+
+def test_call_update_state_first(setup_llm_model) -> None:
+    llm, model = setup_llm_model
+
+    chat_settings = ChatPromptSettings(
+        system_messages=AIAgentConversation(
+            messages=(
+                SystemMessage(content="Test message 1"),
+                HumanMessage(content="Test message 2"),
+                AIMessage(content="Test message 3"),
+            ),
+        ),
+        initial_prompt="Initial test message",
+        temperature=1.0,
+    )
+
+    solution_generator = SolutionGenerator(
+        llm=llm,
+        ai_model=model,
+        ai_model_agent=chat_settings,
+    )
+
+    with pytest.raises(AssertionError):
+        solution_generator.generate_solution()
+
+
 def test_get_code_from_solution():
     assert (
         SolutionGenerator.get_code_from_solution(