From 30a7391b9b1f7478cc501fd978c6fa4c65d4a19a Mon Sep 17 00:00:00 2001 From: Stan Girard Date: Wed, 11 Sep 2024 17:04:05 +0200 Subject: [PATCH] wip --- backend/api/quivr_api/logger.py | 2 +- .../quivr_api/middlewares/auth/auth_bearer.py | 1 + .../middlewares/auth/jwt_token_handler.py | 1 + .../models/brains_subscription_invitations.py | 1 + .../analytics/controller/analytics_routes.py | 1 + .../modules/analytics/entity/analytics.py | 3 +- .../api_key/controller/api_key_routes.py | 1 + .../api_key/service/api_key_service.py | 1 + .../modules/assistant/controller/__init__.py | 1 - .../assistant/controller/assistant_routes.py | 4 +-- .../modules/assistant/entity/__init__.py | 0 .../modules/assistant/entity/task_entity.py | 32 +++++++++++++++++++ .../modules/assistant/repository/__init__.py | 0 .../repository/interfaces/__init__.py | 0 .../repository/interfaces/task_interface.py | 23 +++++++++++++ .../modules/brain/controller/__init__.py | 1 - .../api/quivr_api/modules/brain/dto/inputs.py | 1 + .../modules/brain/integrations/Big/Brain.py | 1 + .../brain/integrations/Claude/Brain.py | 1 + .../modules/brain/integrations/GPT4/Brain.py | 1 + .../modules/brain/integrations/Proxy/Brain.py | 1 + .../modules/brain/integrations/SQL/Brain.py | 1 + .../modules/brain/integrations/Self/Brain.py | 3 +- .../service/brain_authorization_service.py | 1 + .../brain/service/brain_user_service.py | 1 + .../service/utils/format_chat_history.py | 1 + .../brain/service/utils/validate_brain.py | 2 -- .../api/quivr_api/modules/chat/dto/chats.py | 1 + .../api/quivr_api/modules/chat/entity/chat.py | 8 ++--- backend/api/quivr_api/modules/dependencies.py | 4 +-- .../modules/models/controller/model_routes.py | 1 + .../prompt/controller/prompt_routes.py | 1 + .../modules/prompt/entity/__init__.py | 4 +-- .../sync/controller/azure_sync_routes.py | 1 + .../sync/controller/github_sync_routes.py | 1 + .../modules/sync/entity/notion_page.py | 1 + .../sync/repository/sync_repository.py | 9 +++--- .../api/quivr_api/modules/tools/__init__.py | 4 +-- .../api/quivr_api/modules/tools/url_reader.py | 1 + .../api/quivr_api/modules/tools/web_search.py | 1 + .../service/generate_file_signed_url.py | 2 +- .../modules/upload/service/list_files.py | 3 +- .../quivr_api/modules/vector/entity/vector.py | 3 +- .../quivr_api/routes/subscription_routes.py | 1 + .../utils/handle_request_validation_error.py | 1 + backend/api/tests/settings/test_settings.py | 5 +-- backend/core/examples/pdf_parsing_tika.py | 7 ++-- backend/core/examples/simple_question.py | 3 +- .../examples/simple_question_streaming.py | 7 ++-- backend/core/quivr_core/brain/brain.py | 2 +- backend/core/quivr_core/chat.py | 3 +- backend/core/quivr_core/llm/llm_endpoint.py | 4 +-- .../core/quivr_core/quivr_rag_langgraph.py | 11 +++---- backend/core/quivr_core/utils.py | 2 +- backend/core/tests/fixture_chunks.py | 1 - .../community/test_markdown_processor.py | 1 - .../core/tests/processor/docx/test_docx.py | 1 - .../processor/epub/test_epub_processor.py | 1 - backend/core/tests/processor/odt/test_odt.py | 1 - .../pdf/test_unstructured_pdf_processor.py | 1 - .../processor/test_default_implementations.py | 2 -- .../processor/test_simple_txt_processor.py | 1 - .../tests/processor/test_tika_processor.py | 1 - backend/core/tests/test_brain.py | 1 - backend/core/tests/test_chat_history.py | 1 - backend/core/tests/test_chat_llm.py | 1 - backend/core/tests/test_llm_endpoint.py | 1 - backend/core/tests/test_utils.py | 1 - backend/worker/tests/conftest.py | 1 - backend/worker/tests/test_utils.py | 1 - 70 files changed, 127 insertions(+), 67 deletions(-) create mode 100644 backend/api/quivr_api/modules/assistant/entity/__init__.py create mode 100644 backend/api/quivr_api/modules/assistant/entity/task_entity.py create mode 100644 backend/api/quivr_api/modules/assistant/repository/__init__.py create mode 100644 backend/api/quivr_api/modules/assistant/repository/interfaces/__init__.py create mode 100644 backend/api/quivr_api/modules/assistant/repository/interfaces/task_interface.py diff --git a/backend/api/quivr_api/logger.py b/backend/api/quivr_api/logger.py index 0fc69cfceec8..b839e9aef4f1 100644 --- a/backend/api/quivr_api/logger.py +++ b/backend/api/quivr_api/logger.py @@ -4,7 +4,7 @@ from colorlog import ( ColoredFormatter, -) # You need to install this package: pip install colorlog +) def get_logger(logger_name, log_file="application.log"): diff --git a/backend/api/quivr_api/middlewares/auth/auth_bearer.py b/backend/api/quivr_api/middlewares/auth/auth_bearer.py index 212ac6034551..b15d9a4a76b9 100644 --- a/backend/api/quivr_api/middlewares/auth/auth_bearer.py +++ b/backend/api/quivr_api/middlewares/auth/auth_bearer.py @@ -3,6 +3,7 @@ from fastapi import Depends, HTTPException, Request from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer + from quivr_api.middlewares.auth.jwt_token_handler import ( decode_access_token, verify_token, diff --git a/backend/api/quivr_api/middlewares/auth/jwt_token_handler.py b/backend/api/quivr_api/middlewares/auth/jwt_token_handler.py index 4e412b41deaf..e6438d2ac448 100644 --- a/backend/api/quivr_api/middlewares/auth/jwt_token_handler.py +++ b/backend/api/quivr_api/middlewares/auth/jwt_token_handler.py @@ -4,6 +4,7 @@ from jose import jwt from jose.exceptions import JWTError + from quivr_api.modules.user.entity.user_identity import UserIdentity SECRET_KEY = os.environ.get("JWT_SECRET_KEY") diff --git a/backend/api/quivr_api/models/brains_subscription_invitations.py b/backend/api/quivr_api/models/brains_subscription_invitations.py index bbe474c22927..fc0ddf048a29 100644 --- a/backend/api/quivr_api/models/brains_subscription_invitations.py +++ b/backend/api/quivr_api/models/brains_subscription_invitations.py @@ -1,6 +1,7 @@ from uuid import UUID from pydantic import BaseModel, ConfigDict + from quivr_api.logger import get_logger logger = get_logger(__name__) diff --git a/backend/api/quivr_api/modules/analytics/controller/analytics_routes.py b/backend/api/quivr_api/modules/analytics/controller/analytics_routes.py index dfb889b38a15..4dc924032d80 100644 --- a/backend/api/quivr_api/modules/analytics/controller/analytics_routes.py +++ b/backend/api/quivr_api/modules/analytics/controller/analytics_routes.py @@ -1,6 +1,7 @@ from uuid import UUID from fastapi import APIRouter, Depends, Query + from quivr_api.middlewares.auth.auth_bearer import AuthBearer, get_current_user from quivr_api.modules.analytics.entity.analytics import Range from quivr_api.modules.analytics.service.analytics_service import AnalyticsService diff --git a/backend/api/quivr_api/modules/analytics/entity/analytics.py b/backend/api/quivr_api/modules/analytics/entity/analytics.py index e7f79cefd83c..6d8dced41f76 100644 --- a/backend/api/quivr_api/modules/analytics/entity/analytics.py +++ b/backend/api/quivr_api/modules/analytics/entity/analytics.py @@ -1,7 +1,8 @@ +from datetime import date from enum import IntEnum from typing import List + from pydantic import BaseModel -from datetime import date class Range(IntEnum): diff --git a/backend/api/quivr_api/modules/api_key/controller/api_key_routes.py b/backend/api/quivr_api/modules/api_key/controller/api_key_routes.py index dedf9f7042cf..7215ae76d80f 100644 --- a/backend/api/quivr_api/modules/api_key/controller/api_key_routes.py +++ b/backend/api/quivr_api/modules/api_key/controller/api_key_routes.py @@ -3,6 +3,7 @@ from uuid import uuid4 from fastapi import APIRouter, Depends + from quivr_api.logger import get_logger from quivr_api.middlewares.auth import AuthBearer, get_current_user from quivr_api.modules.api_key.dto.outputs import ApiKeyInfo diff --git a/backend/api/quivr_api/modules/api_key/service/api_key_service.py b/backend/api/quivr_api/modules/api_key/service/api_key_service.py index f459eff387f4..9290a51e4dc8 100644 --- a/backend/api/quivr_api/modules/api_key/service/api_key_service.py +++ b/backend/api/quivr_api/modules/api_key/service/api_key_service.py @@ -1,6 +1,7 @@ from datetime import datetime from fastapi import HTTPException + from quivr_api.logger import get_logger from quivr_api.modules.api_key.repository.api_key_interface import ApiKeysInterface from quivr_api.modules.api_key.repository.api_keys import ApiKeys diff --git a/backend/api/quivr_api/modules/assistant/controller/__init__.py b/backend/api/quivr_api/modules/assistant/controller/__init__.py index 94bc92dd8166..cc8eb3907031 100644 --- a/backend/api/quivr_api/modules/assistant/controller/__init__.py +++ b/backend/api/quivr_api/modules/assistant/controller/__init__.py @@ -1,7 +1,6 @@ # noqa: from .assistant_routes import assistant_router - __all__ = [ "assistant_router", ] diff --git a/backend/api/quivr_api/modules/assistant/controller/assistant_routes.py b/backend/api/quivr_api/modules/assistant/controller/assistant_routes.py index 80fbe95192da..5876ae5d88d8 100644 --- a/backend/api/quivr_api/modules/assistant/controller/assistant_routes.py +++ b/backend/api/quivr_api/modules/assistant/controller/assistant_routes.py @@ -1,9 +1,9 @@ -from fastapi import APIRouter, Request, Depends +from fastapi import APIRouter, Depends, Request + from quivr_api.logger import get_logger from quivr_api.middlewares.auth.auth_bearer import AuthBearer, get_current_user from quivr_api.modules.user.entity.user_identity import UserIdentity - logger = get_logger(__name__) diff --git a/backend/api/quivr_api/modules/assistant/entity/__init__.py b/backend/api/quivr_api/modules/assistant/entity/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/api/quivr_api/modules/assistant/entity/task_entity.py b/backend/api/quivr_api/modules/assistant/entity/task_entity.py new file mode 100644 index 000000000000..24bfdf8e2e13 --- /dev/null +++ b/backend/api/quivr_api/modules/assistant/entity/task_entity.py @@ -0,0 +1,32 @@ +from datetime import datetime +from uuid import UUID + +from sqlmodel import TIMESTAMP, Column, Field, SQLModel, text +from sqlmodel import UUID as PGUUID + + +class Task(SQLModel, table=True): + __tablename__ = "tasks" # type: ignore + + id: UUID | None = Field( + default=None, + sa_column=Column( + PGUUID, + server_default=text("uuid_generate_v4()"), + primary_key=True, + ), + ) + + pretty_id: str + user_id: UUID = Field(foreign_key="users.id") + status: str + creation_time: datetime | None = Field( + default=None, + sa_column=Column( + TIMESTAMP(timezone=False), + server_default=text("CURRENT_TIMESTAMP"), + ), + ) + # Json for answer_raw + answer_raw: dict | None = Field(default=None) + answer_pretty: str | None = Field(default=None) diff --git a/backend/api/quivr_api/modules/assistant/repository/__init__.py b/backend/api/quivr_api/modules/assistant/repository/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/api/quivr_api/modules/assistant/repository/interfaces/__init__.py b/backend/api/quivr_api/modules/assistant/repository/interfaces/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/backend/api/quivr_api/modules/assistant/repository/interfaces/task_interface.py b/backend/api/quivr_api/modules/assistant/repository/interfaces/task_interface.py new file mode 100644 index 000000000000..fa2092d8c801 --- /dev/null +++ b/backend/api/quivr_api/modules/assistant/repository/interfaces/task_interface.py @@ -0,0 +1,23 @@ +from abc import ABC, abstractmethod +from typing import List +from uuid import UUID + +from quivr_api.modules.assistant.entity.task_entity import Task + + +class TasksInterface(ABC): + @abstractmethod + def create_task(self, task: Task) -> Task: + pass + + @abstractmethod + def get_task_by_id(self, task_id: UUID) -> Task: + pass + + @abstractmethod + def delete_task(self, task_id: UUID) -> None: + pass + + @abstractmethod + def get_tasks_by_user_id(self, user_id: UUID) -> List[Task]: + pass diff --git a/backend/api/quivr_api/modules/brain/controller/__init__.py b/backend/api/quivr_api/modules/brain/controller/__init__.py index cba0a7e1c6e3..98f5cd9dcf74 100644 --- a/backend/api/quivr_api/modules/brain/controller/__init__.py +++ b/backend/api/quivr_api/modules/brain/controller/__init__.py @@ -1,6 +1,5 @@ from .brain_routes import brain_router - __all__ = [ "brain_router", ] diff --git a/backend/api/quivr_api/modules/brain/dto/inputs.py b/backend/api/quivr_api/modules/brain/dto/inputs.py index 632cd9794797..bbdbb1801e61 100644 --- a/backend/api/quivr_api/modules/brain/dto/inputs.py +++ b/backend/api/quivr_api/modules/brain/dto/inputs.py @@ -2,6 +2,7 @@ from uuid import UUID from pydantic import BaseModel + from quivr_api.logger import get_logger from quivr_api.modules.brain.entity.brain_entity import BrainType from quivr_api.modules.brain.entity.integration_brain import IntegrationType diff --git a/backend/api/quivr_api/modules/brain/integrations/Big/Brain.py b/backend/api/quivr_api/modules/brain/integrations/Big/Brain.py index 0c7c61297b48..141f7de7cf27 100644 --- a/backend/api/quivr_api/modules/brain/integrations/Big/Brain.py +++ b/backend/api/quivr_api/modules/brain/integrations/Big/Brain.py @@ -11,6 +11,7 @@ SystemMessagePromptTemplate, ) from langchain_core.prompts.prompt import PromptTemplate + from quivr_api.logger import get_logger from quivr_api.modules.brain.knowledge_brain_qa import KnowledgeBrainQA from quivr_api.modules.chat.dto.chats import ChatQuestion diff --git a/backend/api/quivr_api/modules/brain/integrations/Claude/Brain.py b/backend/api/quivr_api/modules/brain/integrations/Claude/Brain.py index 14cf0023617b..25732779e4b3 100644 --- a/backend/api/quivr_api/modules/brain/integrations/Claude/Brain.py +++ b/backend/api/quivr_api/modules/brain/integrations/Claude/Brain.py @@ -4,6 +4,7 @@ from langchain_community.chat_models import ChatLiteLLM from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder + from quivr_api.modules.brain.knowledge_brain_qa import KnowledgeBrainQA from quivr_api.modules.chat.dto.chats import ChatQuestion diff --git a/backend/api/quivr_api/modules/brain/integrations/GPT4/Brain.py b/backend/api/quivr_api/modules/brain/integrations/GPT4/Brain.py index f643de0656da..0083b48cc7bd 100644 --- a/backend/api/quivr_api/modules/brain/integrations/GPT4/Brain.py +++ b/backend/api/quivr_api/modules/brain/integrations/GPT4/Brain.py @@ -10,6 +10,7 @@ from langchain_openai import ChatOpenAI from langgraph.graph import END, StateGraph from langgraph.prebuilt import ToolExecutor, ToolInvocation + from quivr_api.logger import get_logger from quivr_api.modules.brain.knowledge_brain_qa import KnowledgeBrainQA from quivr_api.modules.chat.dto.chats import ChatQuestion diff --git a/backend/api/quivr_api/modules/brain/integrations/Proxy/Brain.py b/backend/api/quivr_api/modules/brain/integrations/Proxy/Brain.py index 2816d6e574a8..4d5baa14207c 100644 --- a/backend/api/quivr_api/modules/brain/integrations/Proxy/Brain.py +++ b/backend/api/quivr_api/modules/brain/integrations/Proxy/Brain.py @@ -4,6 +4,7 @@ from langchain_community.chat_models import ChatLiteLLM from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder + from quivr_api.logger import get_logger from quivr_api.modules.brain.knowledge_brain_qa import KnowledgeBrainQA from quivr_api.modules.chat.dto.chats import ChatQuestion diff --git a/backend/api/quivr_api/modules/brain/integrations/SQL/Brain.py b/backend/api/quivr_api/modules/brain/integrations/SQL/Brain.py index c039dcac18f7..12a01d4fbbd1 100644 --- a/backend/api/quivr_api/modules/brain/integrations/SQL/Brain.py +++ b/backend/api/quivr_api/modules/brain/integrations/SQL/Brain.py @@ -7,6 +7,7 @@ from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import RunnablePassthrough + from quivr_api.modules.brain.integrations.SQL.SQL_connector import SQLConnector from quivr_api.modules.brain.knowledge_brain_qa import KnowledgeBrainQA from quivr_api.modules.brain.repository.integration_brains import IntegrationBrain diff --git a/backend/api/quivr_api/modules/brain/integrations/Self/Brain.py b/backend/api/quivr_api/modules/brain/integrations/Self/Brain.py index 65d911e2647e..6a992f6871e4 100644 --- a/backend/api/quivr_api/modules/brain/integrations/Self/Brain.py +++ b/backend/api/quivr_api/modules/brain/integrations/Self/Brain.py @@ -12,13 +12,14 @@ from langchain_core.pydantic_v1 import Field as FieldV1 from langchain_openai import ChatOpenAI from langgraph.graph import END, StateGraph +from typing_extensions import TypedDict + from quivr_api.logger import get_logger from quivr_api.modules.brain.knowledge_brain_qa import KnowledgeBrainQA from quivr_api.modules.chat.dto.chats import ChatQuestion from quivr_api.modules.chat.dto.outputs import GetChatHistoryOutput from quivr_api.modules.chat.service.chat_service import ChatService from quivr_api.modules.dependencies import get_service -from typing_extensions import TypedDict # Post-processing diff --git a/backend/api/quivr_api/modules/brain/service/brain_authorization_service.py b/backend/api/quivr_api/modules/brain/service/brain_authorization_service.py index 1a04511611c1..9583c12396b0 100644 --- a/backend/api/quivr_api/modules/brain/service/brain_authorization_service.py +++ b/backend/api/quivr_api/modules/brain/service/brain_authorization_service.py @@ -2,6 +2,7 @@ from uuid import UUID from fastapi import Depends, HTTPException, status + from quivr_api.middlewares.auth.auth_bearer import get_current_user from quivr_api.modules.brain.entity.brain_entity import RoleEnum from quivr_api.modules.brain.service.brain_service import BrainService diff --git a/backend/api/quivr_api/modules/brain/service/brain_user_service.py b/backend/api/quivr_api/modules/brain/service/brain_user_service.py index b1bf15038723..031cfb8a3351 100644 --- a/backend/api/quivr_api/modules/brain/service/brain_user_service.py +++ b/backend/api/quivr_api/modules/brain/service/brain_user_service.py @@ -2,6 +2,7 @@ from uuid import UUID from fastapi import HTTPException + from quivr_api.logger import get_logger from quivr_api.modules.brain.entity.brain_entity import ( BrainEntity, diff --git a/backend/api/quivr_api/modules/brain/service/utils/format_chat_history.py b/backend/api/quivr_api/modules/brain/service/utils/format_chat_history.py index a66cfab5ebd8..0b3d3c795d98 100644 --- a/backend/api/quivr_api/modules/brain/service/utils/format_chat_history.py +++ b/backend/api/quivr_api/modules/brain/service/utils/format_chat_history.py @@ -1,6 +1,7 @@ from typing import List, Tuple from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage + from quivr_api.modules.chat.dto.outputs import GetChatHistoryOutput diff --git a/backend/api/quivr_api/modules/brain/service/utils/validate_brain.py b/backend/api/quivr_api/modules/brain/service/utils/validate_brain.py index 43ec8e0254db..e69de29bb2d1 100644 --- a/backend/api/quivr_api/modules/brain/service/utils/validate_brain.py +++ b/backend/api/quivr_api/modules/brain/service/utils/validate_brain.py @@ -1,2 +0,0 @@ -from fastapi import HTTPException -from quivr_api.modules.brain.dto.inputs import CreateBrainProperties diff --git a/backend/api/quivr_api/modules/chat/dto/chats.py b/backend/api/quivr_api/modules/chat/dto/chats.py index e900602d15c1..a04a6dcc72e0 100644 --- a/backend/api/quivr_api/modules/chat/dto/chats.py +++ b/backend/api/quivr_api/modules/chat/dto/chats.py @@ -3,6 +3,7 @@ from uuid import UUID from pydantic import BaseModel + from quivr_api.modules.chat.dto.outputs import GetChatHistoryOutput from quivr_api.modules.notification.entity.notification import Notification diff --git a/backend/api/quivr_api/modules/chat/entity/chat.py b/backend/api/quivr_api/modules/chat/entity/chat.py index f989f37f3d71..965b38da8069 100644 --- a/backend/api/quivr_api/modules/chat/entity/chat.py +++ b/backend/api/quivr_api/modules/chat/entity/chat.py @@ -2,12 +2,12 @@ from typing import List from uuid import UUID -from quivr_api.modules.brain.entity.brain_entity import Brain -from quivr_api.modules.user.entity.user_identity import User from sqlalchemy.ext.asyncio import AsyncAttrs -from sqlmodel import JSON, TIMESTAMP +from sqlmodel import JSON, TIMESTAMP, Column, Field, Relationship, SQLModel, text from sqlmodel import UUID as PGUUID -from sqlmodel import Column, Field, Relationship, SQLModel, text + +from quivr_api.modules.brain.entity.brain_entity import Brain +from quivr_api.modules.user.entity.user_identity import User class Chat(SQLModel, table=True): diff --git a/backend/api/quivr_api/modules/dependencies.py b/backend/api/quivr_api/modules/dependencies.py index edb6f728b60e..fd71696cd151 100644 --- a/backend/api/quivr_api/modules/dependencies.py +++ b/backend/api/quivr_api/modules/dependencies.py @@ -7,8 +7,7 @@ from langchain_community.embeddings.ollama import OllamaEmbeddings # from langchain_community.vectorstores.supabase import SupabaseVectorStore -from langchain_openai import OpenAIEmbeddings -from langchain_openai import AzureOpenAIEmbeddings +from langchain_openai import AzureOpenAIEmbeddings, OpenAIEmbeddings # from quivr_api.modules.vector.service.vector_service import VectorService # from quivr_api.modules.vectorstore.supabase import CustomSupabaseVectorStore @@ -22,7 +21,6 @@ from quivr_api.models.settings import BrainSettings from supabase.client import AsyncClient, Client, create_async_client, create_client - # Global variables to store the Supabase client and database instances _supabase_client: Optional[Client] = None _supabase_async_client: Optional[AsyncClient] = None diff --git a/backend/api/quivr_api/modules/models/controller/model_routes.py b/backend/api/quivr_api/modules/models/controller/model_routes.py index a5370c90fa38..75b649a331b3 100644 --- a/backend/api/quivr_api/modules/models/controller/model_routes.py +++ b/backend/api/quivr_api/modules/models/controller/model_routes.py @@ -1,6 +1,7 @@ from typing import Annotated, List from fastapi import APIRouter, Depends + from quivr_api.logger import get_logger from quivr_api.middlewares.auth import AuthBearer, get_current_user from quivr_api.modules.dependencies import get_service diff --git a/backend/api/quivr_api/modules/prompt/controller/prompt_routes.py b/backend/api/quivr_api/modules/prompt/controller/prompt_routes.py index 3aa5b6c75704..82e25a4bfb8a 100644 --- a/backend/api/quivr_api/modules/prompt/controller/prompt_routes.py +++ b/backend/api/quivr_api/modules/prompt/controller/prompt_routes.py @@ -1,6 +1,7 @@ from uuid import UUID from fastapi import APIRouter, Depends + from quivr_api.middlewares.auth import AuthBearer from quivr_api.modules.prompt.entity.prompt import ( CreatePromptProperties, diff --git a/backend/api/quivr_api/modules/prompt/entity/__init__.py b/backend/api/quivr_api/modules/prompt/entity/__init__.py index 1700ebd8133d..324aeee09ffc 100644 --- a/backend/api/quivr_api/modules/prompt/entity/__init__.py +++ b/backend/api/quivr_api/modules/prompt/entity/__init__.py @@ -1,7 +1,7 @@ from .prompt import ( + CreatePromptProperties, + DeletePromptResponse, Prompt, PromptStatusEnum, - CreatePromptProperties, PromptUpdatableProperties, - DeletePromptResponse, ) diff --git a/backend/api/quivr_api/modules/sync/controller/azure_sync_routes.py b/backend/api/quivr_api/modules/sync/controller/azure_sync_routes.py index c905fb5ba88c..2f40c140c52b 100644 --- a/backend/api/quivr_api/modules/sync/controller/azure_sync_routes.py +++ b/backend/api/quivr_api/modules/sync/controller/azure_sync_routes.py @@ -4,6 +4,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import HTMLResponse from msal import ConfidentialClientApplication + from quivr_api.logger import get_logger from quivr_api.middlewares.auth import AuthBearer, get_current_user from quivr_api.modules.sync.dto.inputs import SyncsUserInput, SyncUserUpdateInput diff --git a/backend/api/quivr_api/modules/sync/controller/github_sync_routes.py b/backend/api/quivr_api/modules/sync/controller/github_sync_routes.py index ecc88a5b3aa6..84599965c6a0 100644 --- a/backend/api/quivr_api/modules/sync/controller/github_sync_routes.py +++ b/backend/api/quivr_api/modules/sync/controller/github_sync_routes.py @@ -3,6 +3,7 @@ import requests from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import HTMLResponse + from quivr_api.logger import get_logger from quivr_api.middlewares.auth import AuthBearer, get_current_user from quivr_api.modules.sync.dto.inputs import SyncsUserInput, SyncUserUpdateInput diff --git a/backend/api/quivr_api/modules/sync/entity/notion_page.py b/backend/api/quivr_api/modules/sync/entity/notion_page.py index f84f89fd26ec..7a42f190250f 100644 --- a/backend/api/quivr_api/modules/sync/entity/notion_page.py +++ b/backend/api/quivr_api/modules/sync/entity/notion_page.py @@ -3,6 +3,7 @@ from uuid import UUID from pydantic import BaseModel, ConfigDict, Field, field_validator + from quivr_api.modules.sync.entity.sync_models import NotionSyncFile diff --git a/backend/api/quivr_api/modules/sync/repository/sync_repository.py b/backend/api/quivr_api/modules/sync/repository/sync_repository.py index f7f7e3bc0347..998e71d7ae3b 100644 --- a/backend/api/quivr_api/modules/sync/repository/sync_repository.py +++ b/backend/api/quivr_api/modules/sync/repository/sync_repository.py @@ -2,6 +2,11 @@ from typing import List, Sequence from uuid import UUID +from sqlalchemy import or_ +from sqlalchemy.exc import IntegrityError +from sqlmodel import col, select +from sqlmodel.ext.asyncio.session import AsyncSession + from quivr_api.logger import get_logger from quivr_api.modules.dependencies import BaseRepository, get_supabase_client from quivr_api.modules.notification.service.notification_service import ( @@ -10,10 +15,6 @@ from quivr_api.modules.sync.dto.inputs import SyncsActiveInput, SyncsActiveUpdateInput from quivr_api.modules.sync.entity.sync_models import NotionSyncFile, SyncsActive from quivr_api.modules.sync.repository.sync_interfaces import SyncInterface -from sqlalchemy import or_ -from sqlalchemy.exc import IntegrityError -from sqlmodel import col, select -from sqlmodel.ext.asyncio.session import AsyncSession notification_service = NotificationService() diff --git a/backend/api/quivr_api/modules/tools/__init__.py b/backend/api/quivr_api/modules/tools/__init__.py index 79767b838703..adb3f960174a 100644 --- a/backend/api/quivr_api/modules/tools/__init__.py +++ b/backend/api/quivr_api/modules/tools/__init__.py @@ -1,4 +1,4 @@ +from .email_sender import EmailSenderTool from .image_generator import ImageGeneratorTool -from .web_search import WebSearchTool from .url_reader import URLReaderTool -from .email_sender import EmailSenderTool +from .web_search import WebSearchTool diff --git a/backend/api/quivr_api/modules/tools/url_reader.py b/backend/api/quivr_api/modules/tools/url_reader.py index e5f05d60e996..e1b1086f8fe7 100644 --- a/backend/api/quivr_api/modules/tools/url_reader.py +++ b/backend/api/quivr_api/modules/tools/url_reader.py @@ -11,6 +11,7 @@ from langchain_community.document_loaders import PlaywrightURLLoader from langchain_core.tools import BaseTool from pydantic import BaseModel + from quivr_api.logger import get_logger logger = get_logger(__name__) diff --git a/backend/api/quivr_api/modules/tools/web_search.py b/backend/api/quivr_api/modules/tools/web_search.py index a7357069ecd9..2c2d004bdc73 100644 --- a/backend/api/quivr_api/modules/tools/web_search.py +++ b/backend/api/quivr_api/modules/tools/web_search.py @@ -10,6 +10,7 @@ from langchain.pydantic_v1 import Field as FieldV1 from langchain_core.tools import BaseTool from pydantic import BaseModel + from quivr_api.logger import get_logger logger = get_logger(__name__) diff --git a/backend/api/quivr_api/modules/upload/service/generate_file_signed_url.py b/backend/api/quivr_api/modules/upload/service/generate_file_signed_url.py index a982991fe215..089e509b9fdd 100644 --- a/backend/api/quivr_api/modules/upload/service/generate_file_signed_url.py +++ b/backend/api/quivr_api/modules/upload/service/generate_file_signed_url.py @@ -1,8 +1,8 @@ +import os from multiprocessing import get_logger from quivr_api.modules.dependencies import get_supabase_client from supabase.client import Client -import os logger = get_logger() diff --git a/backend/api/quivr_api/modules/upload/service/list_files.py b/backend/api/quivr_api/modules/upload/service/list_files.py index bf03756e94b3..b6a4abd8f2fb 100644 --- a/backend/api/quivr_api/modules/upload/service/list_files.py +++ b/backend/api/quivr_api/modules/upload/service/list_files.py @@ -1,8 +1,7 @@ from multiprocessing import get_logger -from supabase.client import Client - from quivr_api.modules.dependencies import get_supabase_client +from supabase.client import Client logger = get_logger() diff --git a/backend/api/quivr_api/modules/vector/entity/vector.py b/backend/api/quivr_api/modules/vector/entity/vector.py index a0d46baa41a8..b0583f64034a 100644 --- a/backend/api/quivr_api/modules/vector/entity/vector.py +++ b/backend/api/quivr_api/modules/vector/entity/vector.py @@ -3,12 +3,11 @@ from pgvector.sqlalchemy import Vector as PGVector from pydantic import BaseModel +from quivr_api.models.settings import settings from sqlalchemy import Column from sqlmodel import JSON, Column, Field, SQLModel, text from sqlmodel import UUID as PGUUID -from quivr_api.models.settings import settings - class Vector(SQLModel, table=True): __tablename__ = "vectors" # type: ignore diff --git a/backend/api/quivr_api/routes/subscription_routes.py b/backend/api/quivr_api/routes/subscription_routes.py index 8619d976c9c5..cecf227c3581 100644 --- a/backend/api/quivr_api/routes/subscription_routes.py +++ b/backend/api/quivr_api/routes/subscription_routes.py @@ -3,6 +3,7 @@ from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel + from quivr_api.logger import get_logger from quivr_api.middlewares.auth.auth_bearer import AuthBearer, get_current_user from quivr_api.models.brains_subscription_invitations import BrainSubscription diff --git a/backend/api/quivr_api/utils/handle_request_validation_error.py b/backend/api/quivr_api/utils/handle_request_validation_error.py index 3c33da4e03f8..d539c7885550 100644 --- a/backend/api/quivr_api/utils/handle_request_validation_error.py +++ b/backend/api/quivr_api/utils/handle_request_validation_error.py @@ -1,6 +1,7 @@ from fastapi import FastAPI, Request, status from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse + from quivr_api.logger import get_logger logger = get_logger(__name__) diff --git a/backend/api/tests/settings/test_settings.py b/backend/api/tests/settings/test_settings.py index f47910b17272..6a89d5b7854c 100644 --- a/backend/api/tests/settings/test_settings.py +++ b/backend/api/tests/settings/test_settings.py @@ -1,7 +1,8 @@ -from unittest.mock import patch, MagicMock -from quivr_api.modules.dependencies import get_embedding_client +from unittest.mock import MagicMock, patch + from langchain_community.embeddings.ollama import OllamaEmbeddings from langchain_openai import AzureOpenAIEmbeddings +from quivr_api.modules.dependencies import get_embedding_client def test_ollama_embedding(): diff --git a/backend/core/examples/pdf_parsing_tika.py b/backend/core/examples/pdf_parsing_tika.py index c84f27f78538..b86a232a292d 100644 --- a/backend/core/examples/pdf_parsing_tika.py +++ b/backend/core/examples/pdf_parsing_tika.py @@ -1,12 +1,11 @@ from langchain_core.embeddings import DeterministicFakeEmbedding from langchain_core.language_models import FakeListChatModel -from rich.console import Console -from rich.panel import Panel -from rich.prompt import Prompt - from quivr_core import Brain from quivr_core.config import LLMEndpointConfig from quivr_core.llm.llm_endpoint import LLMEndpoint +from rich.console import Console +from rich.panel import Panel +from rich.prompt import Prompt if __name__ == "__main__": brain = Brain.from_files( diff --git a/backend/core/examples/simple_question.py b/backend/core/examples/simple_question.py index 99886c277b5e..3b6ce3b5d4bf 100644 --- a/backend/core/examples/simple_question.py +++ b/backend/core/examples/simple_question.py @@ -1,7 +1,8 @@ import tempfile + from quivr_core import Brain -from quivr_core.quivr_rag_langgraph import QuivrQARAGLangGraph from quivr_core.quivr_rag import QuivrQARAG +from quivr_core.quivr_rag_langgraph import QuivrQARAGLangGraph if __name__ == "__main__": with tempfile.NamedTemporaryFile(mode="w", suffix=".txt") as temp_file: diff --git a/backend/core/examples/simple_question_streaming.py b/backend/core/examples/simple_question_streaming.py index 324a1a2d24ef..34b0ba710287 100644 --- a/backend/core/examples/simple_question_streaming.py +++ b/backend/core/examples/simple_question_streaming.py @@ -1,9 +1,10 @@ -from dotenv import load_dotenv -import tempfile import asyncio +import tempfile + +from dotenv import load_dotenv from quivr_core import Brain -from quivr_core.quivr_rag_langgraph import QuivrQARAGLangGraph from quivr_core.quivr_rag import QuivrQARAG +from quivr_core.quivr_rag_langgraph import QuivrQARAGLangGraph async def main(): diff --git a/backend/core/quivr_core/brain/brain.py b/backend/core/quivr_core/brain/brain.py index ad9d265e2d57..14d8ad80bc33 100644 --- a/backend/core/quivr_core/brain/brain.py +++ b/backend/core/quivr_core/brain/brain.py @@ -2,7 +2,7 @@ import logging from pathlib import Path from pprint import PrettyPrinter -from typing import Any, AsyncGenerator, Callable, Dict, Self, Union, Type +from typing import Any, AsyncGenerator, Callable, Dict, Self, Type, Union from uuid import UUID, uuid4 from langchain_core.documents import Document diff --git a/backend/core/quivr_core/chat.py b/backend/core/quivr_core/chat.py index 01ab56fe00f4..90697eecc8e4 100644 --- a/backend/core/quivr_core/chat.py +++ b/backend/core/quivr_core/chat.py @@ -3,6 +3,7 @@ from uuid import UUID, uuid4 from langchain_core.messages import AIMessage, HumanMessage + from quivr_core.models import ChatMessage @@ -54,7 +55,7 @@ def iter_pairs(self) -> Generator[Tuple[HumanMessage, AIMessage], None, None]: """ # Reverse the chat_history, newest first it = iter(self.get_chat_history(newest_first=True)) - for ai_message, human_message in zip(it, it): + for ai_message, human_message in zip(it, it, strict=False): assert isinstance( human_message.msg, HumanMessage ), f"msg {human_message} is not HumanMessage" diff --git a/backend/core/quivr_core/llm/llm_endpoint.py b/backend/core/quivr_core/llm/llm_endpoint.py index bb6aea86699f..a05e3c1c0795 100644 --- a/backend/core/quivr_core/llm/llm_endpoint.py +++ b/backend/core/quivr_core/llm/llm_endpoint.py @@ -1,10 +1,10 @@ import logging from urllib.parse import parse_qs, urlparse +from langchain_anthropic import ChatAnthropic from langchain_core.language_models.chat_models import BaseChatModel -from pydantic.v1 import SecretStr from langchain_openai import AzureChatOpenAI, ChatOpenAI -from langchain_anthropic import ChatAnthropic +from pydantic.v1 import SecretStr from quivr_core.brain.info import LLMInfo from quivr_core.config import LLMEndpointConfig diff --git a/backend/core/quivr_core/quivr_rag_langgraph.py b/backend/core/quivr_core/quivr_rag_langgraph.py index 8e4e457b3956..f856e52ceac7 100644 --- a/backend/core/quivr_core/quivr_rag_langgraph.py +++ b/backend/core/quivr_core/quivr_rag_langgraph.py @@ -1,16 +1,15 @@ import logging -from typing import AsyncGenerator, Optional, Sequence, Annotated, Sequence, TypedDict +from typing import Annotated, AsyncGenerator, Optional, Sequence, TypedDict # TODO(@aminediro): this is the only dependency to langchain package, we should remove it from langchain.retrievers import ContextualCompressionRetriever from langchain_core.callbacks import Callbacks from langchain_core.documents import BaseDocumentCompressor, Document -from langchain_core.messages import AIMessage, HumanMessage, BaseMessage +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage from langchain_core.messages.ai import AIMessageChunk from langchain_core.vectorstores import VectorStore - -from langgraph.graph.message import add_messages from langgraph.graph import END, StateGraph +from langgraph.graph.message import add_messages from quivr_core.chat import ChatHistory from quivr_core.config import RAGConfig @@ -22,12 +21,12 @@ RAGResponseMetadata, cited_answer, ) -from quivr_core.prompts import CONDENSE_QUESTION_PROMPT, ANSWER_PROMPT +from quivr_core.prompts import ANSWER_PROMPT, CONDENSE_QUESTION_PROMPT from quivr_core.utils import ( + combine_documents, format_file_list, get_chunk_metadata, parse_chunk_response, - combine_documents, parse_response, ) diff --git a/backend/core/quivr_core/utils.py b/backend/core/quivr_core/utils.py index beddb2d2df01..38f8c51c54f6 100644 --- a/backend/core/quivr_core/utils.py +++ b/backend/core/quivr_core/utils.py @@ -152,7 +152,7 @@ def combine_documents( docs, document_prompt=DEFAULT_DOCUMENT_PROMPT, document_separator="\n\n" ): # for each docs, add an index in the metadata to be able to cite the sources - for doc, index in zip(docs, range(len(docs))): + for doc, index in zip(docs, range(len(docs)), strict=False): doc.metadata["index"] = index doc_strings = [format_document(doc, document_prompt) for doc in docs] return document_separator.join(doc_strings) diff --git a/backend/core/tests/fixture_chunks.py b/backend/core/tests/fixture_chunks.py index 9481462d0063..47f0e28d21c5 100644 --- a/backend/core/tests/fixture_chunks.py +++ b/backend/core/tests/fixture_chunks.py @@ -5,7 +5,6 @@ from langchain_core.embeddings import DeterministicFakeEmbedding from langchain_core.messages.ai import AIMessageChunk from langchain_core.vectorstores import InMemoryVectorStore - from quivr_core.chat import ChatHistory from quivr_core.config import LLMEndpointConfig, RAGConfig from quivr_core.llm import LLMEndpoint diff --git a/backend/core/tests/processor/community/test_markdown_processor.py b/backend/core/tests/processor/community/test_markdown_processor.py index 063b07c0fc7b..8e06b5602cd1 100644 --- a/backend/core/tests/processor/community/test_markdown_processor.py +++ b/backend/core/tests/processor/community/test_markdown_processor.py @@ -2,7 +2,6 @@ from uuid import uuid4 import pytest - from quivr_core.files.file import FileExtension, QuivrFile from quivr_core.processor.implementations.default import MarkdownProcessor diff --git a/backend/core/tests/processor/docx/test_docx.py b/backend/core/tests/processor/docx/test_docx.py index ecbbeef7e9e1..a4d445a37bb3 100644 --- a/backend/core/tests/processor/docx/test_docx.py +++ b/backend/core/tests/processor/docx/test_docx.py @@ -2,7 +2,6 @@ from uuid import uuid4 import pytest - from quivr_core.files.file import FileExtension, QuivrFile from quivr_core.processor.implementations.default import DOCXProcessor diff --git a/backend/core/tests/processor/epub/test_epub_processor.py b/backend/core/tests/processor/epub/test_epub_processor.py index ae9afc986187..a81c569bf917 100644 --- a/backend/core/tests/processor/epub/test_epub_processor.py +++ b/backend/core/tests/processor/epub/test_epub_processor.py @@ -2,7 +2,6 @@ from uuid import uuid4 import pytest - from quivr_core.files.file import FileExtension, QuivrFile from quivr_core.processor.implementations.default import EpubProcessor diff --git a/backend/core/tests/processor/odt/test_odt.py b/backend/core/tests/processor/odt/test_odt.py index 899b4fd39f1f..bc7b565ac182 100644 --- a/backend/core/tests/processor/odt/test_odt.py +++ b/backend/core/tests/processor/odt/test_odt.py @@ -2,7 +2,6 @@ from uuid import uuid4 import pytest - from quivr_core.files.file import FileExtension, QuivrFile from quivr_core.processor.implementations.default import ODTProcessor diff --git a/backend/core/tests/processor/pdf/test_unstructured_pdf_processor.py b/backend/core/tests/processor/pdf/test_unstructured_pdf_processor.py index bd03f9b123c5..a9d3ed2371f1 100644 --- a/backend/core/tests/processor/pdf/test_unstructured_pdf_processor.py +++ b/backend/core/tests/processor/pdf/test_unstructured_pdf_processor.py @@ -2,7 +2,6 @@ from uuid import uuid4 import pytest - from quivr_core.files.file import FileExtension, QuivrFile from quivr_core.processor.implementations.default import UnstructuredPDFProcessor diff --git a/backend/core/tests/processor/test_default_implementations.py b/backend/core/tests/processor/test_default_implementations.py index 9248b57d254f..62489b34748b 100644 --- a/backend/core/tests/processor/test_default_implementations.py +++ b/backend/core/tests/processor/test_default_implementations.py @@ -1,5 +1,4 @@ import pytest - from quivr_core.files.file import FileExtension from quivr_core.processor.processor_base import ProcessorBase @@ -7,7 +6,6 @@ @pytest.mark.base def test___build_processor(): from langchain_community.document_loaders.base import BaseLoader - from quivr_core.processor.implementations.default import _build_processor cls = _build_processor("TestCLS", BaseLoader, [FileExtension.txt]) diff --git a/backend/core/tests/processor/test_simple_txt_processor.py b/backend/core/tests/processor/test_simple_txt_processor.py index 126ee1cac59f..cf075b47d381 100644 --- a/backend/core/tests/processor/test_simple_txt_processor.py +++ b/backend/core/tests/processor/test_simple_txt_processor.py @@ -1,6 +1,5 @@ import pytest from langchain_core.documents import Document - from quivr_core.files.file import FileExtension from quivr_core.processor.implementations.simple_txt_processor import ( SimpleTxtProcessor, diff --git a/backend/core/tests/processor/test_tika_processor.py b/backend/core/tests/processor/test_tika_processor.py index 8b99f10e897e..c1a69cd30723 100644 --- a/backend/core/tests/processor/test_tika_processor.py +++ b/backend/core/tests/processor/test_tika_processor.py @@ -1,5 +1,4 @@ import pytest - from quivr_core.processor.implementations.tika_processor import TikaProcessor # TODO: TIKA server should be set diff --git a/backend/core/tests/test_brain.py b/backend/core/tests/test_brain.py index 93eb1c350df7..367df9e07044 100644 --- a/backend/core/tests/test_brain.py +++ b/backend/core/tests/test_brain.py @@ -4,7 +4,6 @@ import pytest from langchain_core.documents import Document from langchain_core.embeddings import Embeddings - from quivr_core.brain import Brain from quivr_core.chat import ChatHistory from quivr_core.llm import LLMEndpoint diff --git a/backend/core/tests/test_chat_history.py b/backend/core/tests/test_chat_history.py index 8cb89e7c8bfd..b5af198a6833 100644 --- a/backend/core/tests/test_chat_history.py +++ b/backend/core/tests/test_chat_history.py @@ -3,7 +3,6 @@ import pytest from langchain_core.messages import AIMessage, HumanMessage - from quivr_core.chat import ChatHistory diff --git a/backend/core/tests/test_chat_llm.py b/backend/core/tests/test_chat_llm.py index 7eeeb97303b6..0af31929496b 100644 --- a/backend/core/tests/test_chat_llm.py +++ b/backend/core/tests/test_chat_llm.py @@ -1,5 +1,4 @@ import pytest - from quivr_core import ChatLLM diff --git a/backend/core/tests/test_llm_endpoint.py b/backend/core/tests/test_llm_endpoint.py index ba5fb79c5a5f..d50f60222f96 100644 --- a/backend/core/tests/test_llm_endpoint.py +++ b/backend/core/tests/test_llm_endpoint.py @@ -3,7 +3,6 @@ import pytest from langchain_core.language_models import FakeListChatModel from pydantic.v1.error_wrappers import ValidationError - from quivr_core.config import LLMEndpointConfig from quivr_core.llm import LLMEndpoint diff --git a/backend/core/tests/test_utils.py b/backend/core/tests/test_utils.py index 66ef21126f5f..7847f94e158c 100644 --- a/backend/core/tests/test_utils.py +++ b/backend/core/tests/test_utils.py @@ -3,7 +3,6 @@ import pytest from langchain_core.messages.ai import AIMessageChunk from langchain_core.messages.tool import ToolCall - from quivr_core.utils import ( get_prev_message_str, model_supports_function_calling, diff --git a/backend/worker/tests/conftest.py b/backend/worker/tests/conftest.py index 596b81c062de..7d9828a365d0 100644 --- a/backend/worker/tests/conftest.py +++ b/backend/worker/tests/conftest.py @@ -2,7 +2,6 @@ from uuid import uuid4 import pytest - from quivr_worker.files import File diff --git a/backend/worker/tests/test_utils.py b/backend/worker/tests/test_utils.py index 3b0cfdab51f5..d0a764037b8d 100644 --- a/backend/worker/tests/test_utils.py +++ b/backend/worker/tests/test_utils.py @@ -4,7 +4,6 @@ import pytest from langchain_core.documents import Document - from quivr_worker.utils import _patch_json