Feature/integration tests

assessment_module_manager/assessment_module_manager/module/request_to_module.py

@@ -33,6 +33,7 @@ async def find_module_by_name(module_name: str) -> Optional[Module]:
     return None
 
 
+# pylint: disable=too-many-positional-arguments
 async def request_to_module(module: Module, headers: dict, path: str, lms_url: str, data: Optional[dict], method: str) -> ModuleResponse:
     """
     Helper function to send a request to a module.

athena/athena/database.py

@@ -10,7 +10,8 @@
 
 # SQLite specific configuration
 is_sqlite = env.DATABASE_URL.startswith("sqlite:///")
-if is_sqlite:
+is_in_memory = env.DATABASE_URL == "sqlite:///:memory:"
+if is_sqlite and not is_in_memory:
     connect_args = {"check_same_thread": False}
     # create the data directory if it does not exist
     data_dir = os.path.dirname(env.DATABASE_URL[10:])

modules/modeling/module_modeling_llm/pyproject.toml

@@ -18,7 +18,7 @@ langsmith = "0.1.106"
 
 [tool.poetry.group.dev.dependencies]
 pydantic = "1.10.17"
-prospector = "1.10.2"
+prospector = "^1.10.2"
 types-requests = "2.31.0.8"
 
 [tool.poetry.scripts]

modules/programming/module_programming_llm/module_programming_llm/helpers/utils.py

@@ -104,6 +104,7 @@ def temporary_remote(remote_name: str, repo: Repo, remote_url: str) -> Iterator[
         repo.delete_remote(remote)
 
 
+# pylint: disable=too-many-positional-arguments
 def get_diff(src_repo: Repo, 
              dst_repo: Repo, 
              src_prefix: str = "a",

scripts/install_modules.py

@@ -15,6 +15,7 @@ def main():
         "modules/programming/module_programming_themisml",
         "modules/programming/module_programming_apted",
         "modules/modeling/module_modeling_llm",
+        "tests/integration_tests"
     ]
 
     success = True

scripts/lock_modules.py

@@ -14,7 +14,8 @@ def main():
         "modules/text/module_text_cofee",
         "modules/programming/module_programming_themisml",
         "modules/programming/module_programming_apted",
-        "modules/modeling/module_modeling_llm"
+        "modules/modeling/module_modeling_llm",
+        "tests/integration_tests"
     ]
 
     success = True

scripts/test_modules.py

@@ -1,29 +1,28 @@
 import subprocess
 import os
 import sys
+import shutil
 
 
 def main():
-    modules = [
-        "docs",
-        "log_viewer",
-        "assessment_module_manager",
-        "athena",  # the version in this commit only, can differ for modules
-        "modules/programming/module_example",
-        "modules/programming/module_programming_llm",
-        "modules/text/module_text_llm",
-        "modules/text/module_text_cofee",
-        "modules/programming/module_programming_themisml",
-        "modules/programming/module_programming_apted",
-        "modules/modeling/module_modeling_llm"
+    poetry_path = shutil.which("poetry")
+    if poetry_path is None:
+        print("Could not find poetry.")
+        sys.exit(1)
+    os.environ["POETRY_PATH"] = poetry_path
+
+    test_modules = [
+        "tests/integration_tests"
     ]
 
     success = True
 
-    if success:
-        sys.exit(0)
-    else:
-        sys.exit(-1)
+    for module in test_modules:
+        result = subprocess.run([poetry_path, "run", "pytest", module])
+        if result.returncode != 0:
+            success = False
+
+    sys.exit(0 if success else -1)
 
 
 if __name__ == "__main__":

tests/integration_tests/.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+  "python.pythonPath": "./.venv/bin/python",
+  "python.analysis.typeCheckingMode": "basic",
+}

tests/integration_tests/README.md

@@ -0,0 +1,17 @@
+# Integration tests
+
+Executes integration tests for Athena
+
+## Development Setup
+
+1. Install dependencies with poetry:
+
+```
+poetry install
+```
+
+## Usage
+
+### Start Directly
+
+`poetry run tests_all` from the top-level Athena directory

tests/integration_tests/integration_tests/mocks/llm_core/__init__.py

@@ -0,0 +1,19 @@
+import importlib.util
+import sys
+import os
+
+utils_path = None
+for path in sys.path:
+    if path.endswith('llm_core'):
+        utils_path = os.path.join(path, 'llm_core', 'utils')
+        break
+
+if utils_path and os.path.exists(utils_path):
+    utils_spec = importlib.util.spec_from_file_location("llm_core.utils", os.path.join(utils_path, '__init__.py'))
+    utils = importlib.util.module_from_spec(utils_spec)
+    sys.modules["llm_core.utils"] = utils
+    utils_spec.loader.exec_module(utils)
+else:
+    raise ModuleNotFoundError("Cannot find the llm_core.utils module in the specified path")
+
+print(utils)

tests/integration_tests/integration_tests/mocks/llm_core/models/__init__.py

@@ -0,0 +1,9 @@
+import sys
+
+from llm_core.models.mock_llm import FakeLLM
+
+ModelConfigType = FakeLLM
+DefaultModelConfig = FakeLLM
+evaluation_model = FakeLLM
+
+sys.path = [p for p in sys.path if 'integration_tests' not in p]

tests/integration_tests/integration_tests/mocks/llm_core/models/mock_llm.py

@@ -0,0 +1,10 @@
+from abc import ABC
+
+from langchain_community.chat_models import FakeListChatModel
+from pydantic import BaseModel
+
+
+class FakeLLM(BaseModel, ABC):
+
+    def get_model(self) -> FakeListChatModel:
+        return FakeListChatModel(responses=["mock"])

tests/integration_tests/integration_tests/runners/run_assessment_module_manager.py

@@ -0,0 +1,103 @@
+from abc import ABC, abstractmethod
+
+import pytest
+import subprocess
+import os
+import sys
+import time
+from threading import Thread
+from queue import Queue, Empty
+
+from langchain_core.language_models import FakeListLLM
+from pydantic import BaseModel
+
+
+class FakeLLMConfig(BaseModel, ABC):
+
+    @abstractmethod
+    def get_model(self) -> FakeListLLM:
+        pass
+
+def enqueue_output(out, queue):
+    for line in iter(out.readline, b''):
+        queue.put(line)
+    out.close()
+
+
+@pytest.fixture
+def run_assessment_module_manager(monkeypatch):
+    monkeypatch.setenv("DATABASE_URL", "sqlite:///../data/integration_test_data.sqlite")
+    monkeypatch.setenv("PRODUCTION", "1")
+
+    # secrets
+    monkeypatch.setenv("LMS_LOCAL_SECRET", "integration test lms local secret")
+    monkeypatch.setenv("MODULE_EXAMPLE_SECRET", "integration test MODULE_EXAMPLE_SECRET")
+    monkeypatch.setenv("MODULE_PROGRAMMING_LLM_SECRET", "integration test MODULE_PROGRAMMING_LLM_SECRET")
+    monkeypatch.setenv("MODULE_TEXT_LLM_SECRET", "integration test MODULE_TEXT_LLM_SECRET")
+    monkeypatch.setenv("MODULE_TEXT_COFEE_SECRET", "integration test MODULE_TEXT_COFEE_SECRET")
+    monkeypatch.setenv("MODULE_PROGRAMMING_THEMISML_SECRET", "integration test MODULE_PROGRAMMING_THEMISML_SECRET")
+    monkeypatch.setenv("MODULE_PROGRAMMING_APTED_SECRET", "integration test MODULE_PROGRAMMING_APTED_SECRET")
+    monkeypatch.setenv("MODULE_MODELING_LLM_SECRET", "integration test MODULE_MODELING_LLM_SECRET")
+
+    poetry_path = os.getenv("POETRY_PATH")
+    if poetry_path is None:
+        raise EnvironmentError("Set POETRY_PATH environment variable to run the test")
+
+    current_cwd = os.getcwd()
+    module_cwd = os.path.join(current_cwd, "assessment_module_manager")
+
+    ON_POSIX = 'posix' in sys.builtin_module_names
+
+    process = subprocess.Popen(
+        [poetry_path, "run", "python", "-m", "assessment_module_manager"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        cwd=module_cwd,
+        close_fds=ON_POSIX,
+        bufsize=1
+    )
+    queue = Queue()
+    # TODO logs are written into the error stream, should be fixed in the future
+    thread = Thread(target=enqueue_output, args=(process.stderr, queue))
+    thread.daemon = True
+    thread.start()
+
+    ready = False
+    stderr_output = ""
+    for _ in range(5):
+        while True:
+            try:
+                current_output = queue.get_nowait()
+            except Empty:
+                break
+            else:
+                stderr_output = stderr_output + current_output
+
+        if "Application startup complete" in stderr_output:
+            ready = True
+            break
+
+        time.sleep(1)
+
+    if not ready:
+        process.terminate()
+        process.wait()
+        stderr_output = stderr_output + process.stderr.read()
+        raise TimeoutError(f"Assessment Module Manager didn't start at time.\nError: {stderr_output}")
+
+    monkeypatch.delenv("DATABASE_URL")
+    monkeypatch.delenv("PRODUCTION")
+    monkeypatch.delenv("LMS_LOCAL_SECRET")
+    monkeypatch.delenv("MODULE_EXAMPLE_SECRET")
+    monkeypatch.delenv("MODULE_PROGRAMMING_LLM_SECRET")
+    monkeypatch.delenv("MODULE_TEXT_LLM_SECRET")
+    monkeypatch.delenv("MODULE_TEXT_COFEE_SECRET")
+    monkeypatch.delenv("MODULE_PROGRAMMING_THEMISML_SECRET")
+    monkeypatch.delenv("MODULE_PROGRAMMING_APTED_SECRET")
+    monkeypatch.delenv("MODULE_MODELING_LLM_SECRET")
+
+    yield process
+
+    process.terminate()
+    process.wait()

tests/integration_tests/integration_tests/runners/run_module_modeling_llm.py

@@ -0,0 +1,90 @@
+from unittest.mock import Mock, patch
+
+import pytest
+import subprocess
+import os
+import sys
+import time
+from threading import Thread
+from queue import Queue, Empty
+
+
+def enqueue_output(out, queue):
+    for line in iter(out.readline, b''):
+        queue.put(line)
+    out.close()
+
+
+@pytest.fixture
+def run_module_modeling_llm(monkeypatch):
+    orig_pythonpath = os.environ.get('PYTHONPATH', '')
+
+    monkeypatch.setenv("DATABASE_URL", "sqlite:///../data/integration_test_data.sqlite")
+    monkeypatch.setenv("PRODUCTION", "1")
+
+    # secrets
+    monkeypatch.setenv("SECRET", "integration test MODULE_MODELING_LLM_SECRET")
+    monkeypatch.setenv("LLM_DEFAULT_MODEL", "fake_model")
+
+    current_cwd = os.getcwd() # global level Athena
+    monkeypatch.setenv('PYTHONPATH', current_cwd + '/tests/integration_tests/integration_tests/mocks:' + os.environ.get('PYTHONPATH', ''))
+
+    poetry_path = os.getenv("POETRY_PATH")
+    if poetry_path is None:
+        raise EnvironmentError("Set POETRY_PATH environment variable to run the test")
+
+    module_cwd = os.path.join(current_cwd, "modules/modeling/module_modeling_llm")
+
+    ON_POSIX = 'posix' in sys.builtin_module_names
+
+    process = subprocess.Popen(
+        [poetry_path, "run", "python", "-m", "module_modeling_llm"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        cwd=module_cwd,
+        close_fds=ON_POSIX,
+        bufsize=1
+    )
+    queue = Queue()
+    # TODO logs are written into the error stream, should be fixed in the future
+    thread = Thread(target=enqueue_output, args=(process.stderr, queue))
+    thread.daemon = True
+    thread.start()
+
+    ready = False
+    stderr_output = ""
+    for _ in range(5):
+        while True:
+            try:
+                current_output = queue.get_nowait()
+            except Empty:
+                break
+            else:
+                if current_output == '':
+                    break
+                stderr_output = stderr_output + current_output
+
+        if "Application startup complete" in stderr_output:
+            ready = True
+            break
+
+        time.sleep(1)
+
+    if not ready:
+        process.terminate()
+        process.wait()
+        stderr_output = stderr_output + process.stderr.read()
+        raise TimeoutError(f"Module Modeling LLM didn't start at time.\nError: {stderr_output}")
+
+    monkeypatch.delenv("DATABASE_URL")
+    monkeypatch.delenv("PRODUCTION")
+    monkeypatch.delenv("SECRET")
+    monkeypatch.delenv("LLM_DEFAULT_MODEL")
+    monkeypatch.setenv('PYTHONPATH', orig_pythonpath)
+
+    yield process
+
+    process.terminate()
+    process.wait()
+