Feature/evaluation

.github/workflows/rag-evaluation.yml

@@ -0,0 +1,87 @@
+name: RAG Evaluation Pipeline
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  evaluate:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.7.1
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v3
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+    - name: Install dependencies
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      run: poetry install --no-interaction --with test
+
+    # Create results directory
+    - name: Create results directory
+      run: mkdir -p tests/results
+
+    - name: Run evaluation tests
+      env:
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      run: |
+        poetry run pytest tests/test_evaluation/test_experiments.py -v
+
+    - name: Upload evaluation results
+      uses: actions/upload-artifact@v3
+      with:
+        name: evaluation-results
+        path: tests/results/evaluation_results.csv
+
+    # Comment on pull request with results
+    - name: Comment PR with Results
+      if: github.event_name == 'pull_request'
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          const fs = require('fs');
+          const results = fs.readFileSync('tests/results/evaluation_results.csv', 'utf8');
+          const lines = results.split('\n');
+          const headers = lines[0].split(',');
+
+          // Create markdown table
+          let tableContent = '| ' + headers.join(' | ') + ' |\n';
+          tableContent += '|' + headers.map(() => '---').join('|') + '|\n';
+
+          // Add data rows
+          for (let i = 1; i < lines.length; i++) {
+            if (lines[i].trim()) {
+              tableContent += '| ' + lines[i].split(',').join(' | ') + ' |\n';
+            }
+          }
+
+          const comment = `## RAG Evaluation Results\n\n${tableContent}\n[Full Results](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`;
+
+          await github.rest.issues.createComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.issue.number,
+            body: comment
+          });

.gitignore

@@ -1,9 +1,64 @@
-/dist
-chat.md
-chat.cursor_rules
-uploads/
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+dist/
+
+
+# Virtual Environment
 .env
-notebooks
-faiss_index.bin
+.venv
+env/
+venv/
+ENV/
+
+# IDE
+.vscode/
+*.swp
+*.swo
+
+# Project specific
+uploads/
+notebooks/
 faiss_indexes/
+faiss_index.bin
+
+# Test specific
+tests/test_data/papers/*
+# tests/test_data/qa_pairs/*
+# tests/test_data/papers_cache.json
+# tests/test_data/qa_cache.json
+tests/test.log
+.pytest_cache/
+
+# Keep test directory structure
+# !tests/test_data/papers/.gitkeep
+# !tests/test_data/qa_pairs/.gitkeep
+
+# Large files
+*.pdf
+*.bin
+*.pt
+*.pth
+*.model
+
+# Data files
+*.json
+!pyproject.json
+!package.json
+!**/config/*.json
+*.csv
+*.xlsx
+*.db
+*.sqlite3
+
+# Logs
+*.log
+logs/
+
+# OS generated files
+
 

pyproject.toml

@@ -15,15 +15,14 @@ asyncpg = "^0.27.0"
 python-multipart = "^0.0.6"
 together = "^0.2.0"
 unstructured = "^0.7.0"
-langchain = "^0.0.335"  # Updated to a version compatible with pydantic v2
 pinecone-client = "^2.2.1"
 faiss-cpu = "^1.7.4"
-openai = "^1.3.3"  # Updated to latest version
+openai = "^1.3.3"
 sentence-transformers = "^2.2.2"
 python-dotenv = "^1.0.0"
 streamlit = "^1.28.1"
 pymupdf = "^1.23.6"
-arxiv = "^2.1.3"
+arxiv = "1.4.8"
 aiofiles = "^23.2.1"
 tiktoken = "^0.8.0"
 huggingface-hub = "^0.26.2"
@@ -32,6 +31,28 @@ beautifulsoup4 = "^4.12.0"
 nltk = "^3.8.1"
 requests = "^2.31.0"
 urllib3 = "^2.0.7"
+PyPDF2 = "^3.0.1"
+transformers = "^4.35.0"
+torch = "^2.1.0"
+llama-index = "^0.11.23"
+llama-index-embeddings-langchain = "^0.2.0"
+llama-index-llms-langchain = "0.4.0"
+text-generation = "^0.7.0"
+langchain = "^0.3.7"
+langchain-huggingface = "^0.1.2"
+langchain-openai = "^0.2.8"
+ragas = "^0.2.5"
+
+[tool.poetry.group.test]
+optional = true
+
+[tool.poetry.group.test.dependencies]
+pytest = "^7.4.0"
+pymongo = "^4.5.0"
+ragas = "^0.2.5"
+tqdm = "^4.66.1"
+pandas = "^2.2.3"
+text-generation = "^0.7.0"
 
 [build-system]
 requires = ["poetry-core"]

researcher/core/__init__.py

@@ -0,0 +1 @@
+# Empty file to make the directory a Python package 

researcher/core/config/__init__.py

@@ -0,0 +1 @@
+

main.py → researcher/core/main.py

@@ -1,7 +1,7 @@
 import uvicorn
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-from routers import document_routes
+from researcher.core.routers import document_routes
 
 app = FastAPI(
     title="ResearchGPT API",

routers/document_routes.py → researcher/core/routers/document_routes.py

@@ -8,7 +8,7 @@
 import aiofiles
 from utils.text_processing import extract_text, chunk_text
 from utils.vector_store import store_chunks, search_similar_chunks
-from utils.rag_pipeline import RAGPipeline
+from researcher.core.utils.rag_pipeline import RAGPipeline
 from config.model_config import ModelProvider
 
 # Set up logging

utils/model_factory.py → researcher/core/utils/model_factory.py

@@ -2,10 +2,10 @@
 import os
 from openai import AsyncOpenAI
 from huggingface_hub import InferenceClient
-from config.model_config import ModelProvider, get_model_config
 from typing import List, Dict
 import json
 import logging
+from researcher.core.config.model_config import ModelProvider, get_model_config
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)

utils/rag_pipeline.py → researcher/core/utils/rag_pipeline.py

@@ -1,7 +1,7 @@
 from typing import List, Dict, Any, Optional
 from langchain.docstore.document import Document
-from utils.model_factory import ModelFactory, ModelProvider
-from utils.search_utils import GoogleSearchTool, QueryAnalyzer, FunctionRegistry
+from researcher.core.utils.model_factory import ModelFactory, ModelProvider
+from researcher.core.utils.search_utils import GoogleSearchTool, QueryAnalyzer, FunctionRegistry
 import json
 import logging
 

utils/search_utils.py → researcher/core/utils/search_utils.py

@@ -1,7 +1,7 @@
 from typing import List, Dict, Any, Optional
 import aiohttp
 import logging
-from config.search_config import SEARCH_CONFIG
+from researcher.core.config.search_config import SEARCH_CONFIG
 import json
 
 logger = logging.getLogger(__name__)