Merge pull request cheshire-cat-ai#774 from cheshire-cat-ai/develop

New in version 1.5.2: - reduced default chunk size and overlap - @nickprock - update Azure LLM and Embedding configs - @zioproto @cristianorevil - add timestamp to conversation history - @zAlweNy26 - update Qdrant client and key env variable for cloud version - @valentimarco - update unit tests - @pieroit - add http endpoint to chat with the Cat (experimental) - @zAlweNy26 - more dependencies update - @dave90 - preserve settings at plugin update - @luca.gobbi - tool usage via JSON - @Pingdred
AlessandroSpallina · Apr 19, 2024 · 02d31dd · 02d31dd
2 parents 4c9fff0 + b1e5d67
commit 02d31dd
Show file tree

Hide file tree

Showing 25 changed files with 194 additions and 161 deletions.
diff --git a/.env.example b/.env.example
@@ -6,6 +6,7 @@ CORE_PORT=1865
 # Qdrant server
 # QDRANT_HOST=localhost
 # QDRANT_PORT=6333
+# QDRANT_API_KEY=<API_KEY>
 
 # Decide to use https / wss secure protocols
 #CORE_USE_SECURE_PROTOCOLS=true

diff --git a/core/cat/experimental/form/cat_form.py b/core/cat/experimental/form/cat_form.py
@@ -71,15 +71,15 @@ def confirm(self) -> bool:
     "confirm": """
 
         # Queries the LLM and check if user is agree or not
-        response = self.cat.llm(confirm_prompt, stream=True)
+        response = self.cat.llm(confirm_prompt)
         return "true" in response.lower()
 
     # Check if the user wants to exit the form
     # it is run at the befginning of every form.next()
     def check_exit_intent(self) -> bool:
 
         # Get user message
-        history = self.stringify_convo_history()
+        user_message = self.cat.working_memory["user_message_json"]["text"]
 
         # Stop examples
         stop_examples = """
@@ -102,17 +102,15 @@ def check_exit_intent(self) -> bool:
 
 {stop_examples}
 
-This is the conversation:
-
-{history}
+User said "{user_message}"
 
 JSON:
 ```json
 {{
     "exit": """
 
         # Queries the LLM and check if user is agree or not
-        response = self.cat.llm(check_exit_prompt, stream=True)
+        response = self.cat.llm(check_exit_prompt)
         return "true" in response.lower()
 
     # Execute the dialogue step

diff --git a/core/cat/factory/embedder.py b/core/cat/factory/embedder.py
@@ -91,7 +91,7 @@ class EmbedderAzureOpenAIConfig(EmbedderSettings):
     openai_api_key: str
     model: str
     azure_endpoint: str
-    openai_api_type: str
+    openai_api_type: str = "azure"
     openai_api_version: str
     deployment: str
 
@@ -132,8 +132,10 @@ class EmbedderCohereConfig(EmbedderSettings):
 
 class EmbedderQdrantFastEmbedConfig(EmbedderSettings):
     model_name: FastEmbedModels = Field(title="Model name", default="BAAI/bge-base-en")
-    max_length: int = 512  # Unknown behavior for values > 512.
-    doc_embed_type: str = "passage"  # as suggest on fastembed documentation, "passage" is the best option for documents.
+    # Unknown behavior for values > 512.
+    max_length: int = 512
+    # as suggest on fastembed documentation, "passage" is the best option for documents.
+    doc_embed_type: str = "passage"
     cache_dir: str = "cat/data/models/fast_embed"
 
     _pyclass: Type = FastEmbedEmbeddings
@@ -154,7 +156,8 @@ class EmbedderGeminiChatConfig(EmbedderSettings):
     """
 
     google_api_key: str
-    model: str = "models/embedding-001"  # Default model https://python.langchain.com/docs/integrations/text_embedding/google_generative_ai
+    # Default model https://python.langchain.com/docs/integrations/text_embedding/google_generative_ai
+    model: str = "models/embedding-001"
 
     _pyclass: Type = GoogleGenerativeAIEmbeddings
 

diff --git a/core/cat/factory/llm.py b/core/cat/factory/llm.py
@@ -1,7 +1,7 @@
-from langchain_community.chat_models import AzureChatOpenAI
+from langchain_openai import AzureChatOpenAI
+from langchain_openai import AzureOpenAI
 from langchain_community.llms import (
     OpenAI,
-    AzureOpenAI,
     Cohere,
     HuggingFaceTextGenInference,
     HuggingFaceEndpoint,
@@ -100,7 +100,7 @@ class LLMOpenAICompatibleConfig(LLMSettings):
 class LLMOpenAIChatConfig(LLMSettings):
     openai_api_key: str
     model_name: str = "gpt-3.5-turbo"
-    temperature: float = 0.7  # default value, from 0 to 1. Higher value create more creative and randomic answers
+    temperature: float = 0.7
     streaming: bool = True
     _pyclass: Type = ChatOpenAI
 
@@ -115,8 +115,8 @@ class LLMOpenAIChatConfig(LLMSettings):
 
 class LLMOpenAIConfig(LLMSettings):
     openai_api_key: str
-    model_name: str = "gpt-3.5-turbo-instruct"  # used instead of text-davinci-003 since it deprecated
-    temperature: float = 0.7  # default value, from 0 to 1. Higher value create more creative and randomic answers
+    model_name: str = "gpt-3.5-turbo-instruct"
+    temperature: float = 0.7
     streaming: bool = True
     _pyclass: Type = OpenAI
 
@@ -134,6 +134,7 @@ class LLMAzureChatOpenAIConfig(LLMSettings):
     openai_api_key: str
     model_name: str = "gpt-35-turbo"  # or gpt-4, use only chat models !
     azure_endpoint: str
+    max_tokens: int = 2048
     openai_api_type: str = "azure"
     # Dont mix api versions https://github.com/hwchase17/langchain/issues/4775
     openai_api_version: str = "2023-05-15"
@@ -155,6 +156,7 @@ class LLMAzureChatOpenAIConfig(LLMSettings):
 class LLMAzureOpenAIConfig(LLMSettings):
     openai_api_key: str
     azure_endpoint: str
+    max_tokens: int = 2048
     api_type: str = "azure"
     # https://learn.microsoft.com/en-us/azure/cognitive-services/openai/reference#completions
     # Current supported versions 2022-12-01, 2023-03-15-preview, 2023-05-15
@@ -208,6 +210,7 @@ class LLMHuggingFaceTextGenInferenceConfig(LLMSettings):
         }
     )
 
+
 # https://api.python.langchain.com/en/latest/llms/langchain_community.llms.huggingface_endpoint.HuggingFaceEndpoint.html
 class LLMHuggingFaceEndpointConfig(LLMSettings):
     endpoint_url: str

diff --git a/core/cat/headers.py b/core/cat/headers.py
@@ -56,10 +56,10 @@ def check_api_key(request: Request, api_key: str = Security(api_key_header)) ->
 
 
 # get or create session (StrayCat)
-def session(request: Request) -> str:
+def session(request: Request) -> StrayCat:
 
     strays = request.app.state.strays
-    user_id = request.headers.get("user_id")
+    user_id = request.headers.get("user_id", "user")
     event_loop = request.app.state.event_loop
 
     if user_id not in strays.keys():

diff --git a/core/cat/looking_glass/agent_manager.py b/core/cat/looking_glass/agent_manager.py
@@ -94,7 +94,7 @@ async def execute_procedures_agent(self, agent_input, stray):
         agent = LLMSingleActionAgent(
             llm_chain=agent_chain,
             output_parser=ChooseProcedureOutputParser(),
-            stop=["\nObservation:"],
+            stop=["}"],
             verbose=self.verbose
         )
 
@@ -359,7 +359,7 @@ def agent_prompt_declarative_memories(self, memory_docs: List[Document]) -> str:
         memory_content = "## Context of documents containing relevant information: " + \
             memories_separator + memories_separator.join(memory_texts)
 
-        # if no data is retrieved from memory don't erite anithing in the prompt
+        # if no data is retrieved from memory don't write anithing in the prompt
         if len(memory_texts) == 0:
             memory_content = ""
 

diff --git a/core/cat/looking_glass/cheshire_cat.py b/core/cat/looking_glass/cheshire_cat.py
@@ -23,19 +23,21 @@
 from cat.rabbit_hole import RabbitHole
 from cat.utils import singleton
 
+
 class Procedure(Protocol):
     name: str
-    procedure_type: str # "tool" or "form"
+    procedure_type: str  # "tool" or "form"
 
     # {
     #   "description": [],
     #   "start_examples": [],
     # }
     triggers_map: Dict[str, List[str]]
 
+
 # main class
 @singleton
-class CheshireCat():
+class CheshireCat:
     """The Cheshire Cat.
 
     This is the main class that manages everything.
@@ -69,7 +71,7 @@ def __init__(self):
         # After memory is loaded, we can get/create tools embeddings
         # every time the mad_hatter finishes syncing hooks, tools and forms, it will notify the Cat (so it can embed tools in vector memory)
         self.mad_hatter.on_finish_plugins_sync_callback = self.embed_procedures
-        self.embed_procedures() # first time launched manually
+        self.embed_procedures()  # first time launched manually
 
         # Agent manager instance (for reasoning)
         self.agent_manager = AgentManager()
@@ -131,6 +133,7 @@ def load_language_model(self) -> BaseLanguageModel:
                 llm = FactoryClass.get_llm_from_config(selected_llm_config["value"])
             except Exception as e:
                 import traceback
+
                 traceback.print_exc()
                 llm = LLMDefaultConfig.get_llm_from_config({})
 
@@ -164,11 +167,16 @@ def load_language_embedder(self) -> embedders.EmbedderSettings:
             FactoryClass = get_embedder_from_name(selected_embedder_class)
 
             # obtain configuration and instantiate Embedder
-            selected_embedder_config = crud.get_setting_by_name(name=selected_embedder_class)
+            selected_embedder_config = crud.get_setting_by_name(
+                name=selected_embedder_class
+            )
             try:
-                embedder = FactoryClass.get_embedder_from_config(selected_embedder_config["value"])
+                embedder = FactoryClass.get_embedder_from_config(
+                    selected_embedder_config["value"]
+                )
             except AttributeError as e:
                 import traceback
+
                 traceback.print_exc()
                 embedder = embedders.EmbedderDumbConfig.get_embedder_from_config({})
             return embedder
@@ -181,24 +189,7 @@ def load_language_embedder(self) -> embedders.EmbedderSettings:
                 }
             )
 
-        # Azure
-        elif type(self._llm) in [AzureOpenAI, AzureChatOpenAI]:
-            embedder = embedders.EmbedderAzureOpenAIConfig.get_embedder_from_config(
-                {
-                    "openai_api_key": self._llm.openai_api_key,
-                    "openai_api_type": "azure",
-                    "model": "text-embedding-ada-002",
-                    # Now the only model for embeddings is text-embedding-ada-002
-                    # It is also possible to use the Azure "deployment" name that is user defined
-                    # when the model is deployed to Azure.
-                    # "deployment": "my-text-embedding-ada-002",
-                    "openai_api_base": self._llm.openai_api_base,
-                    # https://learn.microsoft.com/en-us/azure/cognitive-services/openai/reference#embeddings
-                    # current supported versions 2022-12-01,2023-03-15-preview, 2023-05-15
-                    # Don't mix api versions https://github.com/hwchase17/langchain/issues/4775
-                    "openai_api_version": "2023-05-15",
-                }
-            )
+        # For Azure avoid automatic embedder selection
 
         # Cohere
         elif type(self._llm) in [Cohere]:
@@ -260,17 +251,18 @@ def build_embedded_procedures_hashes(self, embedded_procedures):
 
         hashes = {}
         for ep in embedded_procedures:
-            #log.warning(ep)
+            # log.warning(ep)
             metadata = ep.payload["metadata"]
             content = ep.payload["page_content"]
             source = metadata["source"]
-            trigger_type = metadata.get("trigger_type", "unsupported") # there may be legacy points with no trigger_type
+            # there may be legacy points with no trigger_type
+            trigger_type = metadata.get("trigger_type", "unsupported")
 
             p_hash = f"{source}.{trigger_type}.{content}"
             hashes[p_hash] = ep.id
-        
+
         return hashes
-    
+
     def build_active_procedures_hashes(self, active_procedures):
 
         hashes = {}
@@ -291,21 +283,31 @@ def embed_procedures(self):
 
         # Retrieve from vectorDB all procedural embeddings
         embedded_procedures = self.memory.vectors.procedural.get_all_points()
-        embedded_procedures_hashes = self.build_embedded_procedures_hashes(embedded_procedures)
-
+        embedded_procedures_hashes = self.build_embedded_procedures_hashes(
+            embedded_procedures
+        )
+
         # Easy access to active procedures in mad_hatter (source of truth!)
-        active_procedures_hashes = self.build_active_procedures_hashes(self.mad_hatter.procedures)
+        active_procedures_hashes = self.build_active_procedures_hashes(
+            self.mad_hatter.procedures
+        )
 
         # points_to_be_kept     = set(active_procedures_hashes.keys()) and set(embedded_procedures_hashes.keys()) not necessary
-        points_to_be_deleted  = set(embedded_procedures_hashes.keys()) - set(active_procedures_hashes.keys())
-        points_to_be_embedded = set(active_procedures_hashes.keys()) - set(embedded_procedures_hashes.keys())
-
-        points_to_be_deleted_ids = [embedded_procedures_hashes[p] for p in points_to_be_deleted]
+        points_to_be_deleted = \
+            set(embedded_procedures_hashes.keys()) - set(active_procedures_hashes.keys())
+        points_to_be_embedded = \
+            set(active_procedures_hashes.keys()) - set(embedded_procedures_hashes.keys())
+
+        points_to_be_deleted_ids = [
+            embedded_procedures_hashes[p] for p in points_to_be_deleted
+        ]
         if points_to_be_deleted_ids:
             log.warning(f"Deleting triggers: {points_to_be_deleted}")
             self.memory.vectors.procedural.delete_points(points_to_be_deleted_ids)
 
-        active_triggers_to_be_embedded = [active_procedures_hashes[p] for p in points_to_be_embedded]
+        active_triggers_to_be_embedded = [
+            active_procedures_hashes[p] for p in points_to_be_embedded
+        ]
         for t in active_triggers_to_be_embedded:
             print(t)
             trigger_embedding = self.embedder.embed_documents([t["content"]])
@@ -317,9 +319,11 @@ def embed_procedures(self):
                     "type": t["type"],
                     "trigger_type": t["trigger_type"],
                     "when": time.time(),
-                }
+                },
+            )
+            log.warning(
+                f"Newly embedded {t['type']} trigger: {t['source']}, {t['trigger_type']}, {t['content']}"
             )
-            log.warning(f"Newly embedded {t['type']} trigger: {t['source']}, {t['trigger_type']}, {t['content']}")
 
     def send_ws_message(self, content: str, msg_type="notification"):
         log.error("No websocket connection open")