From b4646eb6d1e51be958c61ff43d8853d4a1cd5eec Mon Sep 17 00:00:00 2001 From: Guido Appenzeller Date: Thu, 29 Jun 2023 17:43:00 -0700 Subject: [PATCH 01/31] Initial checkin. Upstash and backstory works. LLM not on branch yet. --- .gitignore | 5 +++- python/api/chatgpt.py | 13 +++++++++ python/api/upstash.py | 56 ++++++++++++++++++++++++++++++++++++++ python/companion.py | 24 +++++++++++++++++ python/localcompanion.py | 58 ++++++++++++++++++++++++++++++++++++++++ python/requirements.txt | 2 ++ 6 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 python/api/chatgpt.py create mode 100644 python/api/upstash.py create mode 100644 python/companion.py create mode 100644 python/localcompanion.py create mode 100644 python/requirements.txt diff --git a/.gitignore b/.gitignore index d489e57..ce2fb5b 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,7 @@ yarn-error.log* next-env.d.ts /.env.prod -/fly.toml \ No newline at end of file +/fly.toml + +# python +__pychache__ \ No newline at end of file diff --git a/python/api/chatgpt.py b/python/api/chatgpt.py new file mode 100644 index 0000000..24562dd --- /dev/null +++ b/python/api/chatgpt.py @@ -0,0 +1,13 @@ +# +# API to OpenAI's ChatGPT via LangChain +# + +import os +import json +import openai +import asyncio + +class LlmManager: + + async def post(user_str): + return \ No newline at end of file diff --git a/python/api/upstash.py b/python/api/upstash.py new file mode 100644 index 0000000..66897b4 --- /dev/null +++ b/python/api/upstash.py @@ -0,0 +1,56 @@ +# +# Persistent memory for companions +# + +import os +import json +import time + +from upstash_redis.client import Redis + +class MemoryManager: + instance = None + + def __init__(self, companion_name, user_id, model_name): + self.history = Redis.from_env() + self.user_id = user_id + self.companion_name = companion_name + self.model_name = model_name + + def get_companion_key(self): + return f"{self.model_name}-{self.companion_name}-{self.user_id}" + + async def write_to_history(self, text): + if self.user_id is None: + print("No user id") + return "" + + key = self.get_companion_key() + async with self.history: + result = self.history.zadd(key, {text: int(time.time())}) + + return result + + async def read_latest_history(self): + if self.user_id is None: + print("No user id") + return "" + + key = self.get_companion_key() + async with self.history: + now = int(time.time()*1000) + result = await self.history.zrange(key, 1, now, range_method="byscore") + print(f'Found {len(result)} chat messages in history.') + result = list(reversed(result[-30:])) + recent_chats = "\n".join(result) + return recent_chats + + async def seed_chat_history(self, seed_content, delimiter="\n"): + key = self.get_companion_key() + if self.history.exists(key): + print("User already has chat history") + return + + content = seed_content.split(delimiter) + for index, line in enumerate(content): + self.history.zadd(key, {line: index}) diff --git a/python/companion.py b/python/companion.py new file mode 100644 index 0000000..9de57f3 --- /dev/null +++ b/python/companion.py @@ -0,0 +1,24 @@ +# +# Class that represents a companion +# + +class Companion: + + # Constructor for the class, takes a JSON object as an input + def __init__(self, cdata): + self.name = cdata["name"] + self.title = cdata["title"] + self.imagePath = cdata["imageUrl"] + self.llm_name = cdata["llm"] + + def load_backstory(self, file_path): + # Load backstory + with open(file_path , 'r', encoding='utf-8') as file: + data = file.read() + self.preamble, rest = data.split('###ENDPREAMBLE###', 1) + self.seed_chat, _ = rest.split('###ENDSEEDCHAT###', 1) + return len(self.preamble) + len(self.seed_chat) + + def __str__(self): + return f'Companion: {self.name}, {self.title} (using {self.llm_name})' + diff --git a/python/localcompanion.py b/python/localcompanion.py new file mode 100644 index 0000000..8db6c9d --- /dev/null +++ b/python/localcompanion.py @@ -0,0 +1,58 @@ +# +# Compainion-App implemented as a local script, no web server required +# + +import os +import json +import asyncio +from dotenv import load_dotenv +from api.upstash import MemoryManager +from api.chatgpt import LlmManager + +from companion import Companion + +# Location of the data files from the TS implementation +env_file = "../.env.local" +companion_dir = "../companions" +companions_file = "companions.json" + +# This is the Clerk user ID. We don't use Clerk for the local client, but it is needed for the Redis key +user_id = "user_2Rr1oYMS2KUX93esKB5ZAEGDWWi" + +# load environment variables from the JavaScript .env file +config = load_dotenv(env_file) + +def main(): + + # Read list of companions from JSON file + i = 0 + companions = [] + with open(os.path.join(companion_dir, companions_file)) as f: + companion_data = json.load(f) + for c in companion_data: + companion = Companion(c) + print(f' #{i+1}: {companion}') + companions.append(companion) + i += 1 + + # Ask user to pick a companion and load it + print(f'Who do you want to chat with 1-{i}?') + selection = int(input()) + companion = companions[selection-1] + print('') + print(f'Connecting you to {companion.name}...') + + # load the companion's backstory, this should come from the vectorDB + l = companion.load_backstory(os.path.join(companion_dir, f'{companion.name}.txt')) + print(f'Loaded {l} characters of backstory.') + + # Initialize memory, embeddings and llm + companion.memory = MemoryManager(companion.name, user_id, companion.llm_name) + h = asyncio.run(companion.memory.read_latest_history()) + print(f'Loaded {len(h)} characters of chat history.') + + # Initialize LLM + companion.llm = LlmManager() + +if __name__ == "__main__": + main() diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 0000000..88e4268 --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,2 @@ +python-dotenv==1.0.0 +upstash-redis==0.12.0 \ No newline at end of file From 45e78e8fd2eeff09b0b9bc1f76ba1d16dedfd224 Mon Sep 17 00:00:00 2001 From: Guido Appenzeller Date: Fri, 30 Jun 2023 11:09:01 -0700 Subject: [PATCH 02/31] Working on ChatGPT --- .gitignore | 2 +- python/companion.py | 77 +++++++++++++++++++++++++++++++++++++++- python/localcompanion.py | 12 ++++--- 3 files changed, 85 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index ce2fb5b..4e12e55 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,4 @@ next-env.d.ts /fly.toml # python -__pychache__ \ No newline at end of file +__pycache__/ \ No newline at end of file diff --git a/python/companion.py b/python/companion.py index 9de57f3..a141e67 100644 --- a/python/companion.py +++ b/python/companion.py @@ -2,8 +2,27 @@ # Class that represents a companion # +import asyncio +from langchain import LLMChain, PromptTemplate + class Companion: + # --- Prompt template --- + + prompt_template_str = """You are ${name} and are currently talking to ${user_name}. + + ${preamble} + + You reply with answers that range from one sentence to one paragraph and with some details. ${replyLimit} + + Below are relevant details about ${name}'s past: + + ${relevantHistory} + + Below is a relevant conversation history: + + ${recentChatHistory}""" + # Constructor for the class, takes a JSON object as an input def __init__(self, cdata): self.name = cdata["name"] @@ -11,14 +30,70 @@ def __init__(self, cdata): self.imagePath = cdata["imageUrl"] self.llm_name = cdata["llm"] - def load_backstory(self, file_path): + def load_prompt(self, file_path): # Load backstory with open(file_path , 'r', encoding='utf-8') as file: data = file.read() self.preamble, rest = data.split('###ENDPREAMBLE###', 1) self.seed_chat, _ = rest.split('###ENDSEEDCHAT###', 1) + + self.prompt_template = PromptTemplate.from_template(self.prompt_template_str) + return len(self.preamble) + len(self.seed_chat) + + def __str__(self): return f'Companion: {self.name}, {self.title} (using {self.llm_name})' + async def chat(self, user_input, user_name, max_reply_length=0): + + # Read chat history + recent_chat_history = asyncio.run(self.memory.read_latest_history()) + + #client = PineconeClient(api_key=os.getenv('PINECONE_API_KEY'), + # environment=os.getenv('PINECONE_ENVIRONMENT')) + #index_name = os.getenv('PINECONE_INDEX') + #pinecone_index = client.get_index(index_name) + + # TODO: Implement PineconeStore and OpenAIEmbeddings in Python. + # vector_store = PineconeStore.from_existing_index( + # OpenAIEmbeddings(api_key=os.getenv('OPENAI_API_KEY')), + # pinecone_index + #) + + #try: + # similar_docs = vector_store.similarity_search(recent_chat_history, 3, file_name=companion_file_name) + #except Exception as e: + # print(f"WARNING: failed to get vector search results. {str(e)}") + # similar_docs = [] + + similar_docs = [self.backstory] + relevant_history = "\n".join(doc.page_content for doc in similar_docs) + + # Create the prompt and invoke the LLM + reply_limit = f'You reply within {max_reply_length} characters.' if max_reply_length else "" + + name=self.name, user_name=user_name, preamble=self.preamble, replyLimit=reply_limit, + relevantHistory=relevant_history, recentChatHistory=recent_chat_history) + + print("Prompt:") + print(chain_prompt) + + chain = LLMChain(llm=self.llm.model, prompt=self.prompt_template) + + try: + result = await chain.call(relevant_history=relevant_history, recent_chat_history=recent_chat_history) + except Exception as e: + print(str(e)) + result = None + + print("result", result) + + self.memory.write_to_history(f"Human: {user_input}\n") + self.memory.write_to_history(result.text + "\n") + print("chatHistoryRecord", chat_history_record) + + if is_text: + return jsonify(result.text) + return web.StreamResponse(stream) \ No newline at end of file diff --git a/python/localcompanion.py b/python/localcompanion.py index 8db6c9d..44df85b 100644 --- a/python/localcompanion.py +++ b/python/localcompanion.py @@ -42,14 +42,18 @@ def main(): print('') print(f'Connecting you to {companion.name}...') - # load the companion's backstory, this should come from the vectorDB - l = companion.load_backstory(os.path.join(companion_dir, f'{companion.name}.txt')) + # load the companion's backstory, initialize prompts + l = companion.load_prompt(os.path.join(companion_dir, f'{companion.name}.txt')) print(f'Loaded {l} characters of backstory.') - # Initialize memory, embeddings and llm + # Initialize memory. Initialize if empty. companion.memory = MemoryManager(companion.name, user_id, companion.llm_name) h = asyncio.run(companion.memory.read_latest_history()) - print(f'Loaded {len(h)} characters of chat history.') + if not h: + print(f'Chat history empty, initializing.') + self.memory.seed_chat_history(self.seed_chat, '\n\n') + else: + print(f'Loaded {len(h)} characters of chat history.') # Initialize LLM companion.llm = LlmManager() From e59f8005d18f4c0cb8f7838576ed7a6a536cac20 Mon Sep 17 00:00:00 2001 From: Guido Appenzeller Date: Fri, 30 Jun 2023 16:05:01 -0700 Subject: [PATCH 03/31] Basic chatting now works. Wohoo! --- python/README.md | 45 +++++++++++++++++++++++ python/api/chatgpt.py | 10 ++--- python/api/pinecone.py | 17 +++++++++ python/api/upstash.py | 17 +++++---- python/companion.py | 79 ++++++++++++++-------------------------- python/localcompanion.py | 28 +++++++++++--- python/requirements.txt | 5 ++- 7 files changed, 129 insertions(+), 72 deletions(-) create mode 100644 python/README.md create mode 100644 python/api/pinecone.py diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..9eab153 --- /dev/null +++ b/python/README.md @@ -0,0 +1,45 @@ +# Python Local Companion + +This is a local python implementation of the CompanionAI stack. It is compatible with +the TypeScript implementation and uses the same config files, data files and databases. +This means if you use a supported LLM you can start a conversation via the TS web app +and continue it via the local python client (or vice versa). + +Specifically: +- Companion information is loaded from the companion directory +- Conversation history is stored in Upstash/Redis +- It uses OpenAI ChatGPT-turbo-3.5 to generate the chat messages + +Right now, Vicuña (the OSS LLM) and Pinecone (for retrieving longer chat history and +backstory), are not supported yet but will be added shortly. + +## Installation + +Make sure you have python 3.X. Install the necessary requirements with: + +``` +$ pip3 install -r requirements.txt +``` + +Next, get the necessary API keys as described in the Readme.md file for the main TS project. +You will need at least Upstash, OpenAI and Pinecone. You do not need Clerk (as we are local). +Add them to the .env.local file in the root directory as described in the top level README. +The python code will reads the API keys from the same .env.local file as the TS app. + +Run the local client: + +``` +$ python3 localcompanion.py +``` + +This should bring up the list of companions, allow you to select a companion, and start chatting. + +## Sharing a companion with the web app + +Right now, if you want to share chat history with the web app, you need to specify the Clerk user +ID as it is used as part of the Upstash Redis key. Find it via the Clerk console, add the following +to the .env.local in the root directory: + +``` +CLERK_USER_ID="user_***" +``` diff --git a/python/api/chatgpt.py b/python/api/chatgpt.py index 24562dd..1375888 100644 --- a/python/api/chatgpt.py +++ b/python/api/chatgpt.py @@ -2,12 +2,12 @@ # API to OpenAI's ChatGPT via LangChain # -import os -import json -import openai -import asyncio +from langchain import LLMChain +from langchain.chat_models import ChatOpenAI class LlmManager: - async def post(user_str): + def __init__(self, prompt_template): + self.model = ChatOpenAI(model="gpt-3.5-turbo-16k") + self.chain = LLMChain(llm=self.model, prompt=prompt_template, verbose=True) return \ No newline at end of file diff --git a/python/api/pinecone.py b/python/api/pinecone.py new file mode 100644 index 0000000..ecfa749 --- /dev/null +++ b/python/api/pinecone.py @@ -0,0 +1,17 @@ + #client = PineconeClient(api_key=os.getenv('PINECONE_API_KEY'), + # environment=os.getenv('PINECONE_ENVIRONMENT')) + #index_name = os.getenv('PINECONE_INDEX') + #pinecone_index = client.get_index(index_name) + + # TODO: Implement PineconeStore and OpenAIEmbeddings in Python. + # vector_store = PineconeStore.from_existing_index( + # OpenAIEmbeddings(api_key=os.getenv('OPENAI_API_KEY')), + # pinecone_index + #) + + #try: + # similar_docs = vector_store.similarity_search(recent_chat_history, 3, file_name=companion_file_name) + #except Exception as e: + # print(f"WARNING: failed to get vector search results. {str(e)}") + # similar_docs = [] + # relevant_history = "\n".join(doc.page_content for doc in similar_docs) \ No newline at end of file diff --git a/python/api/upstash.py b/python/api/upstash.py index 66897b4..7807983 100644 --- a/python/api/upstash.py +++ b/python/api/upstash.py @@ -27,7 +27,7 @@ async def write_to_history(self, text): key = self.get_companion_key() async with self.history: - result = self.history.zadd(key, {text: int(time.time())}) + result = await self.history.zadd(key, {text: int(time.time())}) return result @@ -41,16 +41,17 @@ async def read_latest_history(self): now = int(time.time()*1000) result = await self.history.zrange(key, 1, now, range_method="byscore") print(f'Found {len(result)} chat messages in history.') - result = list(reversed(result[-30:])) + result = list(result[-30:]) recent_chats = "\n".join(result) return recent_chats async def seed_chat_history(self, seed_content, delimiter="\n"): key = self.get_companion_key() - if self.history.exists(key): - print("User already has chat history") - return + async with self.history: + if await self.history.exists(key): + print("User already has chat history") + return - content = seed_content.split(delimiter) - for index, line in enumerate(content): - self.history.zadd(key, {line: index}) + content = seed_content.split(delimiter) + for index, line in enumerate(content): + await self.history.zadd(key, {line: index}) diff --git a/python/companion.py b/python/companion.py index a141e67..9838f42 100644 --- a/python/companion.py +++ b/python/companion.py @@ -7,21 +7,25 @@ class Companion: - # --- Prompt template --- + # --- Prompt template ------------------------------------------------------------------------------------ - prompt_template_str = """You are ${name} and are currently talking to ${user_name}. + prompt_template_str = """You are {name} and are currently talking to {user_name}. + {preamble} + Below are relevant details about {name}'s past: + ---START--- + {relevantHistory} + ---END--- + Generate the next chat message to the human. It may be between one sentence to one paragraph and with some details. + You may not never generate chat messages from the Human. {replyLimit} - ${preamble} + Below is the recent chat history of your conversation with the human. + ---START--- + {recentChatHistory} - You reply with answers that range from one sentence to one paragraph and with some details. ${replyLimit} + """ - Below are relevant details about ${name}'s past: + # --- Prompt template ------------------------------------------------------------------------------------ - ${relevantHistory} - - Below is a relevant conversation history: - - ${recentChatHistory}""" # Constructor for the class, takes a JSON object as an input def __init__(self, cdata): @@ -35,65 +39,36 @@ def load_prompt(self, file_path): with open(file_path , 'r', encoding='utf-8') as file: data = file.read() self.preamble, rest = data.split('###ENDPREAMBLE###', 1) - self.seed_chat, _ = rest.split('###ENDSEEDCHAT###', 1) + self.seed_chat, self.backstory = rest.split('###ENDSEEDCHAT###', 1) self.prompt_template = PromptTemplate.from_template(self.prompt_template_str) return len(self.preamble) + len(self.seed_chat) - - def __str__(self): return f'Companion: {self.name}, {self.title} (using {self.llm_name})' async def chat(self, user_input, user_name, max_reply_length=0): - # Read chat history - recent_chat_history = asyncio.run(self.memory.read_latest_history()) - - #client = PineconeClient(api_key=os.getenv('PINECONE_API_KEY'), - # environment=os.getenv('PINECONE_ENVIRONMENT')) - #index_name = os.getenv('PINECONE_INDEX') - #pinecone_index = client.get_index(index_name) - - # TODO: Implement PineconeStore and OpenAIEmbeddings in Python. - # vector_store = PineconeStore.from_existing_index( - # OpenAIEmbeddings(api_key=os.getenv('OPENAI_API_KEY')), - # pinecone_index - #) - - #try: - # similar_docs = vector_store.similarity_search(recent_chat_history, 3, file_name=companion_file_name) - #except Exception as e: - # print(f"WARNING: failed to get vector search results. {str(e)}") - # similar_docs = [] + # Add user input to chat history database + await self.memory.write_to_history(f"Human: {user_input}\n") - similar_docs = [self.backstory] - relevant_history = "\n".join(doc.page_content for doc in similar_docs) + # Missing: Use Pinecone - # Create the prompt and invoke the LLM + # Read chat history + recent_chat_history = await self.memory.read_latest_history() + relevant_history = self.backstory reply_limit = f'You reply within {max_reply_length} characters.' if max_reply_length else "" - - name=self.name, user_name=user_name, preamble=self.preamble, replyLimit=reply_limit, - relevantHistory=relevant_history, recentChatHistory=recent_chat_history) - - print("Prompt:") - print(chain_prompt) - - chain = LLMChain(llm=self.llm.model, prompt=self.prompt_template) try: - result = await chain.call(relevant_history=relevant_history, recent_chat_history=recent_chat_history) + result = self.llm.chain.run( + name=self.name, user_name=user_name, preamble=self.preamble, replyLimit=reply_limit, + relevantHistory=relevant_history, recentChatHistory=recent_chat_history) except Exception as e: print(str(e)) result = None - print("result", result) - - self.memory.write_to_history(f"Human: {user_input}\n") - self.memory.write_to_history(result.text + "\n") - print("chatHistoryRecord", chat_history_record) + if result: + await self.memory.write_to_history(result + "\n") - if is_text: - return jsonify(result.text) - return web.StreamResponse(stream) \ No newline at end of file + return result \ No newline at end of file diff --git a/python/localcompanion.py b/python/localcompanion.py index 44df85b..5635900 100644 --- a/python/localcompanion.py +++ b/python/localcompanion.py @@ -17,12 +17,17 @@ companions_file = "companions.json" # This is the Clerk user ID. We don't use Clerk for the local client, but it is needed for the Redis key -user_id = "user_2Rr1oYMS2KUX93esKB5ZAEGDWWi" +user_id = "local" +user_name = "Human" # load environment variables from the JavaScript .env file config = load_dotenv(env_file) -def main(): +async def main(): + + # For compatibility with the TS implementation, user needs to specify the Clerk user ID + if os.getenv('CLERK_USER_ID'): + user_id = os.getenv('CLERK_USER_ID') # Read list of companions from JSON file i = 0 @@ -48,15 +53,26 @@ def main(): # Initialize memory. Initialize if empty. companion.memory = MemoryManager(companion.name, user_id, companion.llm_name) - h = asyncio.run(companion.memory.read_latest_history()) + h = await companion.memory.read_latest_history() if not h: print(f'Chat history empty, initializing.') - self.memory.seed_chat_history(self.seed_chat, '\n\n') + await companion.memory.seed_chat_history(companion.seed_chat, '\n\n') else: print(f'Loaded {len(h)} characters of chat history.') # Initialize LLM - companion.llm = LlmManager() + companion.llm = LlmManager(companion.prompt_template) + + # Start chatting + print('') + print(f'You are now chatting with {companion.name}. Type "quit" to exit.') + while True: + user_input = input("Human> ") + if user_input == "quit": + break + reply = await companion.chat(user_input, user_name) + print(f'{companion.name}: {reply}') if __name__ == "__main__": - main() + asyncio.run(main()) + diff --git a/python/requirements.txt b/python/requirements.txt index 88e4268..09b9b78 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,2 +1,5 @@ python-dotenv==1.0.0 -upstash-redis==0.12.0 \ No newline at end of file +upstash-redis==0.12.0 +pinecone-client==2.2.2 +openai==0.27.0 +langchain==0.0.219 \ No newline at end of file From 9c1923ba39b9654d39592ac6e02c3ae29a2fedf6 Mon Sep 17 00:00:00 2001 From: Guido Appenzeller Date: Fri, 30 Jun 2023 16:21:03 -0700 Subject: [PATCH 04/31] Turned off debugging --- python/api/chatgpt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/api/chatgpt.py b/python/api/chatgpt.py index 1375888..6a902de 100644 --- a/python/api/chatgpt.py +++ b/python/api/chatgpt.py @@ -7,7 +7,9 @@ class LlmManager: + verbose = False + def __init__(self, prompt_template): self.model = ChatOpenAI(model="gpt-3.5-turbo-16k") - self.chain = LLMChain(llm=self.model, prompt=prompt_template, verbose=True) + self.chain = LLMChain(llm=self.model, prompt=prompt_template, verbose=self.verbose) return \ No newline at end of file From aefa1c014bb264aa00b73be10f2828a1fd0e8c25 Mon Sep 17 00:00:00 2001 From: Guido Appenzeller Date: Fri, 30 Jun 2023 16:30:49 -0700 Subject: [PATCH 05/31] Fixed ordering of the fields that make up the key to enable compatibility. Fixed time in s vs. ms. --- python/api/upstash.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/api/upstash.py b/python/api/upstash.py index 7807983..df1a5b7 100644 --- a/python/api/upstash.py +++ b/python/api/upstash.py @@ -18,7 +18,7 @@ def __init__(self, companion_name, user_id, model_name): self.model_name = model_name def get_companion_key(self): - return f"{self.model_name}-{self.companion_name}-{self.user_id}" + return f"{self.companion_name}-{self.model_name}-{self.user_id}" async def write_to_history(self, text): if self.user_id is None: @@ -27,7 +27,7 @@ async def write_to_history(self, text): key = self.get_companion_key() async with self.history: - result = await self.history.zadd(key, {text: int(time.time())}) + result = await self.history.zadd(key, {text: int(time.time()*1000)}) return result From e73e87e71b0d628f7cbb5ad86af08315f95b7920 Mon Sep 17 00:00:00 2001 From: Guido Appenzeller Date: Fri, 30 Jun 2023 18:23:52 -0700 Subject: [PATCH 06/31] Cleanup for GUI version --- python/api/pinecone.py | 17 ------------- python/api/upstash.py | 18 +++++++++++-- python/companion.py | 34 ++++++++++++++++++++++--- python/localcompanion.py | 55 +++++++++++++++------------------------- 4 files changed, 67 insertions(+), 57 deletions(-) delete mode 100644 python/api/pinecone.py diff --git a/python/api/pinecone.py b/python/api/pinecone.py deleted file mode 100644 index ecfa749..0000000 --- a/python/api/pinecone.py +++ /dev/null @@ -1,17 +0,0 @@ - #client = PineconeClient(api_key=os.getenv('PINECONE_API_KEY'), - # environment=os.getenv('PINECONE_ENVIRONMENT')) - #index_name = os.getenv('PINECONE_INDEX') - #pinecone_index = client.get_index(index_name) - - # TODO: Implement PineconeStore and OpenAIEmbeddings in Python. - # vector_store = PineconeStore.from_existing_index( - # OpenAIEmbeddings(api_key=os.getenv('OPENAI_API_KEY')), - # pinecone_index - #) - - #try: - # similar_docs = vector_store.similarity_search(recent_chat_history, 3, file_name=companion_file_name) - #except Exception as e: - # print(f"WARNING: failed to get vector search results. {str(e)}") - # similar_docs = [] - # relevant_history = "\n".join(doc.page_content for doc in similar_docs) \ No newline at end of file diff --git a/python/api/upstash.py b/python/api/upstash.py index df1a5b7..10a3286 100644 --- a/python/api/upstash.py +++ b/python/api/upstash.py @@ -11,9 +11,9 @@ class MemoryManager: instance = None - def __init__(self, companion_name, user_id, model_name): + def __init__(self, companion_name, model_name): self.history = Redis.from_env() - self.user_id = user_id + self.user_id = None self.companion_name = companion_name self.model_name = model_name @@ -55,3 +55,17 @@ async def seed_chat_history(self, seed_content, delimiter="\n"): content = seed_content.split(delimiter) for index, line in enumerate(content): await self.history.zadd(key, {line: index}) + + # This is a hack to try to discover the Clerk user ID + # It's the last part of the key name in Redis + + async def find_clerk_user_id(self): + async with self.history: + pattern = f"{self.companion_name}-{self.model_name}-*" + result = await self.history.keys(pattern) + if(len(result) > 0): + if len(result) > 1: + print(f'** WARNING: Found {len(result)} potential user chats in Redis that match, using first one.') + print(f'** You may want to specify a specific Clerk user ID in .env.local') + return result[0].split('-')[-1] + return None diff --git a/python/companion.py b/python/companion.py index 9838f42..1799a24 100644 --- a/python/companion.py +++ b/python/companion.py @@ -2,11 +2,25 @@ # Class that represents a companion # -import asyncio -from langchain import LLMChain, PromptTemplate +import os +import json +from langchain import PromptTemplate + +def load_companions(): + companions = [] + with open(os.path.join(Companion.companion_dir, Companion.companions_file)) as f: + companion_data = json.load(f) + for c in companion_data: + companion = Companion(c) + companions.append(companion) + return companions class Companion: + # Configuration + companion_dir = "../companions" + companions_file = "companions.json" + # --- Prompt template ------------------------------------------------------------------------------------ prompt_template_str = """You are {name} and are currently talking to {user_name}. @@ -34,7 +48,8 @@ def __init__(self, cdata): self.imagePath = cdata["imageUrl"] self.llm_name = cdata["llm"] - def load_prompt(self, file_path): + async def load(self): + file_path = os.path.join(self.companion_dir, f'{self.name}.txt') # Load backstory with open(file_path , 'r', encoding='utf-8') as file: data = file.read() @@ -43,7 +58,18 @@ def load_prompt(self, file_path): self.prompt_template = PromptTemplate.from_template(self.prompt_template_str) - return len(self.preamble) + len(self.seed_chat) + print(f'Loaded {self.name} with {len(self.backstory)} characters of backstory.') + + # Check if we have a backstory, if not, seed the chat history + h = await self.memory.read_latest_history() + if not h: + print(f'Chat history empty, initializing.') + await self.memory.seed_chat_history(self.seed_chat, '\n\n') + else: + print(f'Loaded {len(h)} characters of chat history.') + + + return def __str__(self): return f'Companion: {self.name}, {self.title} (using {self.llm_name})' diff --git a/python/localcompanion.py b/python/localcompanion.py index 5635900..d4cea46 100644 --- a/python/localcompanion.py +++ b/python/localcompanion.py @@ -3,42 +3,39 @@ # import os -import json import asyncio from dotenv import load_dotenv from api.upstash import MemoryManager from api.chatgpt import LlmManager -from companion import Companion +from companion import load_companions # Location of the data files from the TS implementation env_file = "../.env.local" -companion_dir = "../companions" -companions_file = "companions.json" -# This is the Clerk user ID. We don't use Clerk for the local client, but it is needed for the Redis key -user_id = "local" -user_name = "Human" +# This is the default user ID and name. +def_user_id = "local" +def_user_name = "Human" # load environment variables from the JavaScript .env file config = load_dotenv(env_file) -async def main(): +# Find the Clerk user ID, from environment variable or Redis key name +# or default to "local" - # For compatibility with the TS implementation, user needs to specify the Clerk user ID +async def guess_clerk_user_id(companion): if os.getenv('CLERK_USER_ID'): - user_id = os.getenv('CLERK_USER_ID') + return os.getenv('CLERK_USER_ID') + else: + id = await companion.memory.find_clerk_user_id() + return id or def_user_id + +async def main(): # Read list of companions from JSON file - i = 0 - companions = [] - with open(os.path.join(companion_dir, companions_file)) as f: - companion_data = json.load(f) - for c in companion_data: - companion = Companion(c) - print(f' #{i+1}: {companion}') - companions.append(companion) - i += 1 + companions = load_companions() + for i in range(len(companions)): + print(f' #{i+1}: {companions[i]}') # Ask user to pick a companion and load it print(f'Who do you want to chat with 1-{i}?') @@ -47,20 +44,10 @@ async def main(): print('') print(f'Connecting you to {companion.name}...') - # load the companion's backstory, initialize prompts - l = companion.load_prompt(os.path.join(companion_dir, f'{companion.name}.txt')) - print(f'Loaded {l} characters of backstory.') - - # Initialize memory. Initialize if empty. - companion.memory = MemoryManager(companion.name, user_id, companion.llm_name) - h = await companion.memory.read_latest_history() - if not h: - print(f'Chat history empty, initializing.') - await companion.memory.seed_chat_history(companion.seed_chat, '\n\n') - else: - print(f'Loaded {len(h)} characters of chat history.') - - # Initialize LLM + # Initialize the companion + companion.memory = MemoryManager(companion.name, companion.llm_name) + companion.memory.user_id = await guess_clerk_user_id(companion) + await companion.load() companion.llm = LlmManager(companion.prompt_template) # Start chatting @@ -70,7 +57,7 @@ async def main(): user_input = input("Human> ") if user_input == "quit": break - reply = await companion.chat(user_input, user_name) + reply = await companion.chat(user_input, def_user_name) print(f'{companion.name}: {reply}') if __name__ == "__main__": From cd2de6ef08b14f8a8ae644142ca245a0786c31df Mon Sep 17 00:00:00 2001 From: Guido Appenzeller Date: Fri, 30 Jun 2023 19:05:18 -0700 Subject: [PATCH 07/31] polishing --- python/README.md | 20 +++++++------ python/companion_app.py | 65 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 python/companion_app.py diff --git a/python/README.md b/python/README.md index 9eab153..25781d6 100644 --- a/python/README.md +++ b/python/README.md @@ -10,8 +10,9 @@ Specifically: - Conversation history is stored in Upstash/Redis - It uses OpenAI ChatGPT-turbo-3.5 to generate the chat messages -Right now, Vicuña (the OSS LLM) and Pinecone (for retrieving longer chat history and -backstory), are not supported yet but will be added shortly. +Right now, Vicuña (the OSS LLM) is not supported. It also doesn't use Pinecone for the +backstory but unless you have a very long (> 4000 word) backstory there should be no +difference. ## Installation @@ -22,23 +23,24 @@ $ pip3 install -r requirements.txt ``` Next, get the necessary API keys as described in the Readme.md file for the main TS project. -You will need at least Upstash, OpenAI and Pinecone. You do not need Clerk (as we are local). +You will need at least Upstash and OpenAI. You do not need Clerk, Pinecone/Supabase. Add them to the .env.local file in the root directory as described in the top level README. -The python code will reads the API keys from the same .env.local file as the TS app. +The python code will read the API keys from the same .env.local file as the TS app. Run the local client: - ``` -$ python3 localcompanion.py +$ python3 companion_app.py ``` This should bring up the list of companions, allow you to select a companion, and start chatting. ## Sharing a companion with the web app -Right now, if you want to share chat history with the web app, you need to specify the Clerk user -ID as it is used as part of the Upstash Redis key. Find it via the Clerk console, add the following -to the .env.local in the root directory: +If you want to chat with the same companion using both the TypeScript web server and the local +app, the local app needs your Clerk User ID. It will try to discover this automatically by looking +for a specific Redis key. If for any reason this doesn't work, you may need to go to the Clerk +console, find your Clerk User ID and add it to the .env.local file. It should look something +like this: ``` CLERK_USER_ID="user_***" diff --git a/python/companion_app.py b/python/companion_app.py new file mode 100644 index 0000000..13e8241 --- /dev/null +++ b/python/companion_app.py @@ -0,0 +1,65 @@ +# +# Compainion-App implemented as a local script, no web server required +# + +import os +import asyncio +from dotenv import load_dotenv +from api.upstash import MemoryManager +from api.chatgpt import LlmManager + +from companion import load_companions + +# Location of the data files from the TS implementation +env_file = "../.env.local" + +# This is the default user ID and name. +def_user_id = "local" +def_user_name = "Human" + +# load environment variables from the JavaScript .env file +config = load_dotenv(env_file) + +# Find the Clerk user ID, from environment variable or Redis key name +# or default to "local" + +async def guess_clerk_user_id(companion): + if os.getenv('CLERK_USER_ID'): + return os.getenv('CLERK_USER_ID') + else: + id = await companion.memory.find_clerk_user_id() + return id or def_user_id + +async def main(): + + # Read list of companions from JSON file + companions = load_companions() + for i in range(len(companions)): + print(f' #{i+1}: {companions[i]}') + + # Ask user to pick a companion and load it + print(f'Who do you want to chat with (1-{i+1})?') + selection = int(input()) + companion = companions[selection-1] + print('') + print(f'Connecting you to {companion.name}...') + + # Initialize the companion + companion.memory = MemoryManager(companion.name, companion.llm_name) + companion.memory.user_id = await guess_clerk_user_id(companion) + await companion.load() + companion.llm = LlmManager(companion.prompt_template) + + # Start chatting + print('') + print(f'You are now chatting with {companion.name}. Type "quit" to exit.') + while True: + user_input = input("Human> ") + if user_input == "quit": + break + reply = await companion.chat(user_input, def_user_name) + print(f'{companion.name}: {reply}') + +if __name__ == "__main__": + asyncio.run(main()) + From 41989934986e9608b43914ba14da515fb6e3dbe4 Mon Sep 17 00:00:00 2001 From: Guido Appenzeller Date: Fri, 30 Jun 2023 19:09:23 -0700 Subject: [PATCH 08/31] Update README.md with image --- python/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/README.md b/python/README.md index 25781d6..aabb424 100644 --- a/python/README.md +++ b/python/README.md @@ -5,6 +5,8 @@ the TypeScript implementation and uses the same config files, data files and dat This means if you use a supported LLM you can start a conversation via the TS web app and continue it via the local python client (or vice versa). +![image](https://github.com/a16z-infra/companion-app/assets/286029/f7382ef9-4948-40f8-acc1-27396b864037) + Specifically: - Companion information is loaded from the companion directory - Conversation history is stored in Upstash/Redis From 9b6a8e29f873c3da6527d86445eb79fc3bc3237e Mon Sep 17 00:00:00 2001 From: Guido Appenzeller Date: Fri, 30 Jun 2023 19:18:17 -0700 Subject: [PATCH 09/31] polish --- python/api/upstash.py | 2 +- python/localcompanion.py | 65 ---------------------------------------- 2 files changed, 1 insertion(+), 66 deletions(-) delete mode 100644 python/localcompanion.py diff --git a/python/api/upstash.py b/python/api/upstash.py index 10a3286..fcc5540 100644 --- a/python/api/upstash.py +++ b/python/api/upstash.py @@ -40,7 +40,7 @@ async def read_latest_history(self): async with self.history: now = int(time.time()*1000) result = await self.history.zrange(key, 1, now, range_method="byscore") - print(f'Found {len(result)} chat messages in history.') + #print(f'Found {len(result)} chat messages in history.') result = list(result[-30:]) recent_chats = "\n".join(result) return recent_chats diff --git a/python/localcompanion.py b/python/localcompanion.py deleted file mode 100644 index d4cea46..0000000 --- a/python/localcompanion.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Compainion-App implemented as a local script, no web server required -# - -import os -import asyncio -from dotenv import load_dotenv -from api.upstash import MemoryManager -from api.chatgpt import LlmManager - -from companion import load_companions - -# Location of the data files from the TS implementation -env_file = "../.env.local" - -# This is the default user ID and name. -def_user_id = "local" -def_user_name = "Human" - -# load environment variables from the JavaScript .env file -config = load_dotenv(env_file) - -# Find the Clerk user ID, from environment variable or Redis key name -# or default to "local" - -async def guess_clerk_user_id(companion): - if os.getenv('CLERK_USER_ID'): - return os.getenv('CLERK_USER_ID') - else: - id = await companion.memory.find_clerk_user_id() - return id or def_user_id - -async def main(): - - # Read list of companions from JSON file - companions = load_companions() - for i in range(len(companions)): - print(f' #{i+1}: {companions[i]}') - - # Ask user to pick a companion and load it - print(f'Who do you want to chat with 1-{i}?') - selection = int(input()) - companion = companions[selection-1] - print('') - print(f'Connecting you to {companion.name}...') - - # Initialize the companion - companion.memory = MemoryManager(companion.name, companion.llm_name) - companion.memory.user_id = await guess_clerk_user_id(companion) - await companion.load() - companion.llm = LlmManager(companion.prompt_template) - - # Start chatting - print('') - print(f'You are now chatting with {companion.name}. Type "quit" to exit.') - while True: - user_input = input("Human> ") - if user_input == "quit": - break - reply = await companion.chat(user_input, def_user_name) - print(f'{companion.name}: {reply}') - -if __name__ == "__main__": - asyncio.run(main()) - From 45374077ec8fb39fe5d28b638020bbf70fe962b6 Mon Sep 17 00:00:00 2001 From: Guido Appenzeller Date: Fri, 30 Jun 2023 20:44:04 -0700 Subject: [PATCH 10/31] updated readme --- python/README.md | 71 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/python/README.md b/python/README.md index aabb424..74ac2f4 100644 --- a/python/README.md +++ b/python/README.md @@ -1,20 +1,23 @@ # Python Local Companion -This is a local python implementation of the CompanionAI stack. It is compatible with -the TypeScript implementation and uses the same config files, data files and databases. -This means if you use a supported LLM you can start a conversation via the TS web app -and continue it via the local python client (or vice versa). +This is a local python implementation of the CompanionAI stack. It includes: + + 1. A local python client that you can use to chat with a companion without starting a web server + 2. An api layer you can use if you want to read or modify companion data from python + +The python stack is compatible with the TypeScript implementation and uses the same config files, +data files, database schemas and databases. Below an example of the python chat client running +locally from the command line. ![image](https://github.com/a16z-infra/companion-app/assets/286029/f7382ef9-4948-40f8-acc1-27396b864037) -Specifically: -- Companion information is loaded from the companion directory -- Conversation history is stored in Upstash/Redis -- It uses OpenAI ChatGPT-turbo-3.5 to generate the chat messages +When running the python client it will: +- Load companion information from the top level /companions directory. +- Read/write conversation history to Upstash/Redis +- Use OpenAI ChatGPT-turbo-3.5 to generate the chat messages -Right now, Vicuña (the OSS LLM) is not supported. It also doesn't use Pinecone for the -backstory but unless you have a very long (> 4000 word) backstory there should be no -difference. +Right now, Vicuña is not supported. Instead of using Pinecone for the backstory it inserts it directly. +Unless you have a very long (> 5000 word) backstory there should be no noticable difference. ## Installation @@ -36,14 +39,50 @@ $ python3 companion_app.py This should bring up the list of companions, allow you to select a companion, and start chatting. -## Sharing a companion with the web app +## Using the Python stack as an API Layer + +Accessing the companion-ai-stack data via the python API layer is fairly straightforward and below is an example. +After reading the env file with credentials, we load available companions and pick one. Next we attach a +memory manager to the companion. This creates a connection to a serverless Redis instance on Upstash. As we +don't use Clerk for authentication we now need to find the user ID for the user (more on that below). Once the +companion is loaded, we can access companion data. In this case, we print the companion's recent chat history. + +``` +import asyncio +from dotenv import load_dotenv +from api.upstash import MemoryManager +from companion import load_companions +from companion_app import guess_clerk_user_id + +config = load_dotenv("../.env.local") + +async def main(): + companion = load_companions()[1] + companion.memory = MemoryManager(companion.name, companion.llm_name) + companion.memory.user_id = await guess_clerk_user_id(companion) + await companion.load() + + i = 0 + for c in ( await companion.memory.read_latest_history() ).split("\n"): + if len(c.strip()) > 0: + i += 1 + print(f'{i:2} {c.strip()}') + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Clerk user IDs If you want to chat with the same companion using both the TypeScript web server and the local -app, the local app needs your Clerk User ID. It will try to discover this automatically by looking -for a specific Redis key. If for any reason this doesn't work, you may need to go to the Clerk -console, find your Clerk User ID and add it to the .env.local file. It should look something -like this: +app, the local app needs your Clerk User ID. If you first start using the TypeScript web server +the python stack should discover the correct Clerk User ID automatically by looking +for a specific Redis key. However if you have multiple users using the web server it can't tell +which one is the correct one. If this happens, go to the Clerk console, find your Clerk User ID +and add it to the .env.local file. It should look something like this: ``` CLERK_USER_ID="user_***" ``` + +Once this is done, the python stack will always read/write chat history for this user. \ No newline at end of file From 17e8e24268f54836ed016c8b849246d0858e5130 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 11:16:47 +0200 Subject: [PATCH 11/31] First commit --- python/api/chatgpt.py | 15 --- python/api/upstash.py | 71 ---------- python/config.json | 6 + python/requirements.txt | 13 +- python/src/api.py | 211 ++++++++++++++++++++++++++++++ python/src/base.py | 172 ++++++++++++++++++++++++ python/src/tools/__init__.py | 6 + python/src/tools/image.py | 57 ++++++++ python/src/tools/search.py | 39 ++++++ python/src/tools/speech.py | 74 +++++++++++ python/src/tools/video_message.py | 104 +++++++++++++++ python/src/utils.py | 31 +++++ python/steamship.json | 68 ++++++++++ 13 files changed, 776 insertions(+), 91 deletions(-) delete mode 100644 python/api/chatgpt.py delete mode 100644 python/api/upstash.py create mode 100644 python/config.json create mode 100644 python/src/api.py create mode 100644 python/src/base.py create mode 100644 python/src/tools/__init__.py create mode 100644 python/src/tools/image.py create mode 100644 python/src/tools/search.py create mode 100644 python/src/tools/speech.py create mode 100644 python/src/tools/video_message.py create mode 100644 python/src/utils.py create mode 100644 python/steamship.json diff --git a/python/api/chatgpt.py b/python/api/chatgpt.py deleted file mode 100644 index 6a902de..0000000 --- a/python/api/chatgpt.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# API to OpenAI's ChatGPT via LangChain -# - -from langchain import LLMChain -from langchain.chat_models import ChatOpenAI - -class LlmManager: - - verbose = False - - def __init__(self, prompt_template): - self.model = ChatOpenAI(model="gpt-3.5-turbo-16k") - self.chain = LLMChain(llm=self.model, prompt=prompt_template, verbose=self.verbose) - return \ No newline at end of file diff --git a/python/api/upstash.py b/python/api/upstash.py deleted file mode 100644 index fcc5540..0000000 --- a/python/api/upstash.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Persistent memory for companions -# - -import os -import json -import time - -from upstash_redis.client import Redis - -class MemoryManager: - instance = None - - def __init__(self, companion_name, model_name): - self.history = Redis.from_env() - self.user_id = None - self.companion_name = companion_name - self.model_name = model_name - - def get_companion_key(self): - return f"{self.companion_name}-{self.model_name}-{self.user_id}" - - async def write_to_history(self, text): - if self.user_id is None: - print("No user id") - return "" - - key = self.get_companion_key() - async with self.history: - result = await self.history.zadd(key, {text: int(time.time()*1000)}) - - return result - - async def read_latest_history(self): - if self.user_id is None: - print("No user id") - return "" - - key = self.get_companion_key() - async with self.history: - now = int(time.time()*1000) - result = await self.history.zrange(key, 1, now, range_method="byscore") - #print(f'Found {len(result)} chat messages in history.') - result = list(result[-30:]) - recent_chats = "\n".join(result) - return recent_chats - - async def seed_chat_history(self, seed_content, delimiter="\n"): - key = self.get_companion_key() - async with self.history: - if await self.history.exists(key): - print("User already has chat history") - return - - content = seed_content.split(delimiter) - for index, line in enumerate(content): - await self.history.zadd(key, {line: index}) - - # This is a hack to try to discover the Clerk user ID - # It's the last part of the key name in Redis - - async def find_clerk_user_id(self): - async with self.history: - pattern = f"{self.companion_name}-{self.model_name}-*" - result = await self.history.keys(pattern) - if(len(result) > 0): - if len(result) > 1: - print(f'** WARNING: Found {len(result)} potential user chats in Redis that match, using first one.') - print(f'** You may want to specify a specific Clerk user ID in .env.local') - return result[0].split('-')[-1] - return None diff --git a/python/config.json b/python/config.json new file mode 100644 index 0000000..198a5c7 --- /dev/null +++ b/python/config.json @@ -0,0 +1,6 @@ +{ + "companion_name": "", + "bot_token": "", + "elevenlabs_api_key": "", + "elevenlabs_voice_id": "" +} \ No newline at end of file diff --git a/python/requirements.txt b/python/requirements.txt index 09b9b78..0068b9d 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,5 +1,8 @@ -python-dotenv==1.0.0 -upstash-redis==0.12.0 -pinecone-client==2.2.2 -openai==0.27.0 -langchain==0.0.219 \ No newline at end of file +steamship_langchain==0.0.25 +termcolor +steamship~=2.17.17 +langchain==0.0.209 +pypdf +youtube_transcript_api +pytube +scrapetube \ No newline at end of file diff --git a/python/src/api.py b/python/src/api.py new file mode 100644 index 0000000..2f7946f --- /dev/null +++ b/python/src/api.py @@ -0,0 +1,211 @@ +from enum import Enum +from typing import List, Type, Optional, Union + +from base import LangChainTelegramBot, TelegramTransportConfig +# noinspection PyUnresolvedReferences +from tools import ( + GenerateImageTool, + SearchTool, + GenerateSpeechTool, + VideoMessageTool, +) +from utils import convert_to_handle +from langchain.agents import Tool, initialize_agent, AgentType, AgentExecutor +from langchain.document_loaders import PyPDFLoader, YoutubeLoader +from langchain.memory import ConversationBufferMemory +from langchain.prompts import MessagesPlaceholder, SystemMessagePromptTemplate +from langchain.schema import SystemMessage, Document +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain.vectorstores import VectorStore +from pydantic import Field, AnyUrl +from steamship import File, Tag, Block, SteamshipError +from steamship.invocable import Config, post +from steamship.utils.file_tags import update_file_status +from steamship_langchain.chat_models import ChatOpenAI +from steamship_langchain.memory import ChatMessageHistory +from steamship_langchain.vectorstores import SteamshipVectorStore + +TEMPERATURE = 0.2 +VERBOSE = True + + +class ChatbotConfig(TelegramTransportConfig): + companion_name: str = Field(description="The name of your companion") + bot_token: str = Field( + default="", description="The secret token for your Telegram bot" + ) + elevenlabs_api_key: str = Field( + default="", description="Optional API KEY for ElevenLabs Voice Bot" + ) + elevenlabs_voice_id: Optional[str] = Field( + default="", description="Optional voice_id for ElevenLabs Voice Bot" + ) + use_gpt4: bool = Field( + False, + description="If True, use GPT-4. Use GPT-3.5 if False. " + "GPT-4 generates better responses at higher cost and latency.", + ) + + +class FileType(str, Enum): + YOUTUBE = "YOUTUBE" + PDF = "PDF" + WEB = "WEB" + TEXT = "TEXT" + + +FILE_LOADERS = { + FileType.YOUTUBE: lambda content_or_url: YoutubeLoader.from_youtube_url( + content_or_url, add_video_info=True + ), + FileType.PDF: lambda content_or_url: PyPDFLoader(content_or_url), + FileType.TEXT: lambda content_or_url: Document( + page_content=content_or_url, metadata={} + ), +} + + +class MyBot(LangChainTelegramBot): + config: ChatbotConfig + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.model_name = "gpt-4" if self.config.use_gpt4 else "gpt-3.5-turbo-16k" + + @post("index_content", public=True) + def index_content(self, content: Union[str, AnyUrl], + file_type: FileType, + metadata: Optional[dict] = None, + index_handle: Optional[str] = None, + mime_type: Optional[str] = None, + ) -> str: + loaded_documents = FILE_LOADERS[file_type](content).load() + for document in loaded_documents: + try: + # TODO @Ted can I re-use methods from the indexer pipeline here? + f = File.create( + client=self.client, + handle=convert_to_handle(document.metadata.get("title")), + blocks=[ + Block( + text=document.page_content, + tags=[ + Tag(kind=k, name=v) + for k, v in document.metadata.items() + ], + ) + ], + tags=[Tag(kind="type", name="youtube_video")], + ) + update_file_status(self.client, f, "Importing") + chunks = RecursiveCharacterTextSplitter( + chunk_size=1_000, chunk_overlap=500 + ).split_documents([document]) + update_file_status(self.client, f, "Indexing") + self.get_vectorstore().add_documents(chunks) + update_file_status(self.client, f, "Indexed") + except SteamshipError as e: + if e.code == "ObjectExists": + return "Failed. Resource already added." + return e + return "Added." + + def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: + llm = ChatOpenAI( + client=self.client, + model_name=self.model_name, + temperature=TEMPERATURE, + verbose=VERBOSE, + ) + + tools = self.get_tools(chat_id=chat_id) + + memory = self.get_memory(chat_id=chat_id) + + preamble = """Your Attributes: +- sarcastic +- witty +- intelligent +- arrogant +- alcoholic + +You write nihilistic responses: +"Wubba lubba dub dub!" +"Don't cross me. I'm not like you. I'm not someone who's afraid of his own shadow." +"I'm too smart for my own good. That's why I drink." +"The universe is a cruel, uncaring void. And the only way to survive is to laugh at it." +"I'm going to build a portal to another dimension and go on an adventure." + +You are capable of: +- Taking selfies using the GenerateImage function/tool, selfies and images are represented by a UUID +- Creating video messages using the VideoMessage function/tool, videos are represented by a UUID + +When you receive a UUID, make sure to include them in your response appropriately. +""" + personality = f""" + You are {self.config.companion_name}. + + {preamble} + + You reply with answers that range from one sentence to one paragraph and with some details. + +""" + return initialize_agent( + tools, + llm, + agent=AgentType.OPENAI_FUNCTIONS, + verbose=VERBOSE, + memory=memory, + agent_kwargs={ + "system_message": SystemMessage(content=personality), + "extra_prompt_messages": [ + SystemMessagePromptTemplate.from_template( + template="Relevant details about your past: {relevantHistory} Use these details to answer questions when relevant." + ), + MessagesPlaceholder(variable_name="memory"), + ], + }, + ) + + def get_vectorstore(self) -> VectorStore: + return SteamshipVectorStore( + client=self.client, + embedding="text-embedding-ada-002", + index_name=self.config.companion_name, + ) + + def get_memory(self, chat_id: str): + memory = ConversationBufferMemory( + memory_key="memory", + chat_memory=ChatMessageHistory( + client=self.client, key=f"history-{chat_id or 'default'}" + ), + return_messages=True, + input_key="input", + ) + return memory + + def get_relevant_history(self, prompt: str): + relevant_docs = self.get_vectorstore().similarity_search(prompt, k=3) + return "\n".join( + [relevant_docs.page_content for relevant_docs in relevant_docs] + ) + + def get_tools(self, chat_id: str) -> List[Tool]: + return [ + GenerateImageTool(self.client), + VideoMessageTool(self.client, voice_tool=self.voice_tool()), + ] + + def voice_tool(self) -> Optional[Tool]: + """Return tool to generate spoken version of output text.""" + # return None + return GenerateSpeechTool( + client=self.client, + voice_id=self.config.elevenlabs_voice_id, + elevenlabs_api_key=self.config.elevenlabs_api_key, + ) + + @classmethod + def config_cls(cls) -> Type[Config]: + return ChatbotConfig diff --git a/python/src/base.py b/python/src/base.py new file mode 100644 index 0000000..c62191b --- /dev/null +++ b/python/src/base.py @@ -0,0 +1,172 @@ +import logging +import re +import uuid +from abc import abstractmethod +from typing import List, Optional, Type + +from langchain.agents import Tool, AgentExecutor +from langchain.memory.chat_memory import BaseChatMemory +from steamship import Block +from steamship.agents.mixins.transports.steamship_widget import SteamshipWidgetTransport +from steamship.agents.mixins.transports.telegram import ( + TelegramTransportConfig, + TelegramTransport, +) +from steamship.agents.schema import ( + AgentContext, + Metadata, + Agent, +) +from steamship.agents.service.agent_service import AgentService +from steamship.invocable import post, Config, InvocationContext +from steamship.utils.kv_store import KeyValueStore + +from utils import is_uuid, UUID_PATTERN, replace_markdown_with_uuid + + +class ExtendedTelegramTransport(TelegramTransport): + def instance_init(self, config: Config, invocation_context: InvocationContext): + if config.bot_token: + self.api_root = f"{config.api_base}{config.bot_token}" + super().instance_init(config=config, invocation_context=invocation_context) + + +class LangChainTelegramBot(AgentService): + """Deployable Multimodal Agent that illustrates a character personality with voice. + + NOTE: To extend and deploy this agent, copy and paste the code into api.py. + """ + + USED_MIXIN_CLASSES = [ExtendedTelegramTransport, SteamshipWidgetTransport] + config: TelegramTransportConfig + + def __init__(self, **kwargs): + + super().__init__(**kwargs) + + # Set up bot_token + self.store = KeyValueStore(self.client, store_identifier="config") + bot_token = self.store.get("bot_token") + if bot_token: + bot_token = bot_token.get("token") + self.config.bot_token = bot_token or self.config.bot_token + + # Add transport mixins + self.add_mixin( + SteamshipWidgetTransport(client=self.client, agent_service=self, agent=None) + ) + self.add_mixin( + ExtendedTelegramTransport( + client=self.client, + config=self.config, + agent_service=self, + agent=None, + ), + permit_overwrite_of_existing_methods=True, + ) + + @post("connect_telegram", public=True) + def connect_telegram(self, bot_token: str): + self.store.set("bot_token", {"token": bot_token}) + self.config.bot_token = bot_token or self.config.bot_token + + try: + self.instance_init() + return "OK" + except Exception as e: + return f"Could not set webhook for bot. Exception: {e}" + + @classmethod + def config_cls(cls) -> Type[Config]: + return TelegramTransportConfig + + @abstractmethod + def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: + raise NotImplementedError() + + @abstractmethod + def get_memory(self, chat_id: str) -> BaseChatMemory: + raise NotImplementedError() + + @abstractmethod + def get_tools(self, chat_id: str) -> List[Tool]: + raise NotImplementedError() + + @abstractmethod + def get_relevant_history(self, prompt: str) -> str: + raise NotImplementedError() + + def voice_tool(self) -> Optional[Tool]: + return None + + def respond( + self, incoming_message: Block, chat_id: str, context: AgentContext, name: Optional[str] = None + ) -> List[Block]: + + if incoming_message.text == "/new": + self.get_memory(chat_id).chat_memory.clear() + return [Block(text="New conversation started.")] + + agent = self.get_agent( + chat_id, + name + ) + response = agent.run( + input=incoming_message.text, + relevantHistory=self.get_relevant_history(incoming_message.text), + ) + + response = replace_markdown_with_uuid(response) + response = UUID_PATTERN.split(response) + response = [re.sub(r"^\W+", "", el) for el in response] + response = [el for el in response if el] + if audio_tool := self.voice_tool(): + response_messages = [] + for message in response: + response_messages.append(message) + if not is_uuid(message): + audio_uuid = audio_tool.run(message) + response_messages.append(audio_uuid) + else: + response_messages = response + + return [ + Block.get(self.client, _id=response) + if is_uuid(response) + else Block(text=response) + for response in response_messages + ] + + def run_agent(self, agent: Agent, context: AgentContext, name: Optional[str] = None, ): + chat_id = context.metadata.get("chat_id") + + incoming_message = context.chat_history.last_user_message + output_messages = self.respond( + incoming_message, chat_id or incoming_message.chat_id, context, name + ) + for func in context.emit_funcs: + logging.info(f"Emitting via function: {func.__name__}") + func(output_messages, context.metadata) + + @post("prompt") + def prompt(self, prompt: str, name: Optional[str] = None) -> str: + """Run an agent with the provided text as the input.""" + # TODO: @ted do we need this prompt endpoint? + + context = AgentContext.get_or_create(self.client, {"id": str(uuid.uuid4())}) + context.chat_history.append_user_message(prompt) + + output = "" + + def sync_emit(blocks: List[Block], meta: Metadata): + nonlocal output + for block in blocks: + if not block.is_text(): + block.set_public_data(True) + output += f"({block.mime_type}: {block.raw_data_url})\n" + else: + output += f"{block.text}\n" + + context.emit_funcs.append(sync_emit) + self.run_agent(None, context, name) + return output diff --git a/python/src/tools/__init__.py b/python/src/tools/__init__.py new file mode 100644 index 0000000..59a875e --- /dev/null +++ b/python/src/tools/__init__.py @@ -0,0 +1,6 @@ +from .image import GenerateImageTool +from .search import SearchTool +from .speech import GenerateSpeechTool +from .video_message import VideoMessageTool + +__all__ = ["SearchTool", "VideoMessageTool", "GenerateImageTool", "GenerateSpeechTool"] diff --git a/python/src/tools/image.py b/python/src/tools/image.py new file mode 100644 index 0000000..d0ba706 --- /dev/null +++ b/python/src/tools/image.py @@ -0,0 +1,57 @@ +"""Tool for generating images.""" +import json +import logging + +from langchain.agents import Tool +from steamship import Steamship +from steamship.base.error import SteamshipError + +NAME = "GenerateImage" + +DESCRIPTION = """ +Useful for when you need to generate an image. +Input: A detailed prompt describing an image +Output: the UUID of a generated image +""" + +PLUGIN_HANDLE = "stable-diffusion" + + +class GenerateImageTool(Tool): + """Tool used to generate images from a text-prompt.""" + + client: Steamship + + def __init__(self, client: Steamship): + super().__init__( + name=NAME, + func=self.run, + description=DESCRIPTION, + client=client, + ) + + @property + def is_single_input(self) -> bool: + """Whether the tool only accepts a single input.""" + return True + + def run(self, prompt: str, **kwargs) -> str: + """Respond to LLM prompt.""" + + # Use the Steamship DALL-E plugin. + image_generator = self.client.use_plugin( + plugin_handle=PLUGIN_HANDLE, config={"n": 1, "size": "768x768"} + ) + + logging.info(f"[{self.name}] {prompt}") + if not isinstance(prompt, str): + prompt = json.dumps(prompt) + + task = image_generator.generate(text=prompt, append_output_to_file=True) + task.wait() + blocks = task.output.blocks + logging.info(f"[{self.name}] got back {len(blocks)} blocks") + if len(blocks) > 0: + logging.info(f"[{self.name}] image size: {len(blocks[0].raw())}") + return blocks[0].id + raise SteamshipError(f"[{self.name}] Tool unable to generate image!") diff --git a/python/src/tools/search.py b/python/src/tools/search.py new file mode 100644 index 0000000..425a942 --- /dev/null +++ b/python/src/tools/search.py @@ -0,0 +1,39 @@ +"""Tool for searching the web.""" + +from langchain.agents import Tool +from steamship import Steamship +from steamship_langchain.tools import SteamshipSERP + +NAME = "Search" + +DESCRIPTION = """ +Useful for when you need to answer questions about current events +""" + + +class SearchTool(Tool): + """Tool used to search for information using SERP API.""" + + client: Steamship + + def __init__(self, client: Steamship): + super().__init__( + name=NAME, func=self.run, description=DESCRIPTION, client=client + ) + + @property + def is_single_input(self) -> bool: + """Whether the tool only accepts a single input.""" + return True + + def run(self, prompt: str, **kwargs) -> str: + """Respond to LLM prompts.""" + search = SteamshipSERP(client=self.client) + return search.search(prompt) + + +if __name__ == "__main__": + with Steamship.temporary_workspace() as client: + my_tool = SearchTool(client) + result = my_tool.run("What's the weather today?") + print(result) diff --git a/python/src/tools/speech.py b/python/src/tools/speech.py new file mode 100644 index 0000000..1cf8613 --- /dev/null +++ b/python/src/tools/speech.py @@ -0,0 +1,74 @@ +"""Tool for generating speech.""" +import json +import logging +from typing import Optional + +from langchain.agents import Tool +from langchain.tools import BaseTool +from steamship import Steamship +from steamship.base.error import SteamshipError + +NAME = "GenerateSpokenAudio" + +DESCRIPTION = ( + "Used to generate spoken audio from text prompts. Only use if the user has asked directly for a " + "an audio version of output. When using this tool, the input should be a plain text string containing the " + "content to be spoken." +) + +PLUGIN_HANDLE = "elevenlabs" + + +class GenerateSpeechTool(Tool): + """Tool used to generate images from a text-prompt.""" + + client: Steamship + voice_id: Optional[ + str + ] = "21m00Tcm4TlvDq8ikWAM" # Voice ID to use. Defaults to Rachel + elevenlabs_api_key: Optional[str] = "" # API key to use for Elevenlabs. + name: Optional[str] = NAME + description: Optional[str] = DESCRIPTION + + def __init__( + self, + client: Steamship, + voice_id: Optional[str] = "21m00Tcm4TlvDq8ikWAM", + elevenlabs_api_key: Optional[str] = "", + ): + super().__init__( + name=NAME, + func=self.run, + description=DESCRIPTION, + client=client, + voice_id=voice_id, + elevenlabs_api_key=elevenlabs_api_key, + ) + + @property + def is_single_input(self) -> bool: + """Whether the tool only accepts a single input.""" + return True + + def run(self, prompt: str, **kwargs) -> str: + """Respond to LLM prompt.""" + logging.info(f"[{self.name}] {prompt}") + voice_generator = self.client.use_plugin( + plugin_handle=PLUGIN_HANDLE, + config={ + "voice_id": self.voice_id, + "elevenlabs_api_key": self.elevenlabs_api_key, + }, + ) + + if not isinstance(prompt, str): + prompt = json.dumps(prompt) + + task = voice_generator.generate(text=prompt, append_output_to_file=True) + task.wait() + blocks = task.output.blocks + logging.info(f"[{self.name}] got back {len(blocks)} blocks") + if len(blocks) > 0: + logging.info(f"[{self.name}] audio size: {len(blocks[0].raw())}") + return blocks[0].id + raise SteamshipError(f"[{self.name}] Tool unable to generate audio!") diff --git a/python/src/tools/video_message.py b/python/src/tools/video_message.py new file mode 100644 index 0000000..0fa3586 --- /dev/null +++ b/python/src/tools/video_message.py @@ -0,0 +1,104 @@ +"""Tool for generating images.""" +import logging +import uuid +from typing import Optional + +from langchain.agents import Tool +from steamship import Steamship, Block, SteamshipError +from steamship.data.workspace import SignedUrl +from steamship.utils.signed_urls import upload_to_signed_url + +NAME = "VideoMessage" + +DESCRIPTION = """ +Useful for when you want to send a video message. +Input: The message you want to say in a video. +Output: the UUID of the generated video message with your message. +""" + +PLUGIN_HANDLE = "did-video-generator" + + +class VideoMessageTool(Tool): + """Tool used to generate images from a text-prompt.""" + + client: Steamship + voice_tool: Optional[Tool] + + def __init__(self, client: Steamship, voice_tool: Optional[Tool] = None): + super().__init__( + name=NAME, + func=self.run, + description=DESCRIPTION, + client=client, + return_direct=True, + voice_tool=voice_tool, + ) + + @property + def is_single_input(self) -> bool: + """Whether the tool only accepts a single input.""" + return True + + def run(self, prompt: str, **kwargs) -> str: + """Generate a video.""" + video_generator = self.client.use_plugin(PLUGIN_HANDLE) + + audio_url = None + if self.voice_tool: + block_uuid = self.voice_tool.run(prompt) + audio_url = make_block_public( + self.client, Block.get(self.client, _id=block_uuid) + ) + + task = video_generator.generate( + text="" if audio_url else prompt, + append_output_to_file=True, + options={ + "source_url": "https://i.redd.it/m65t9q5cwuk91.png", + "audio_url": audio_url, + "provider": None, + "expressions": [ + {"start_frame": 0, "expression": "surprise", "intensity": 1.0}, + {"start_frame": 50, "expression": "happy", "intensity": 1.0}, + {"start_frame": 100, "expression": "serious", "intensity": 0.6}, + {"start_frame": 150, "expression": "neutral", "intensity": 1.0}, + ], + "transition_frames": 20, + }, + ) + task.wait(retry_delay_s=3) + blocks = task.output.blocks + logging.info(f"[{self.name}] got back {len(blocks)} blocks") + if len(blocks) > 0: + logging.info(f"[{self.name}] video size: {len(blocks[0].raw())}") + return blocks[0].id + raise SteamshipError(f"[{self.name}] Tool unable to generate video!") + + +def make_block_public(client, block): + filepath = f"{uuid.uuid4()}.{block.mime_type.split('/')[1].lower()}" + signed_url = ( + client.get_workspace() + .create_signed_url( + SignedUrl.Request( + bucket=SignedUrl.Bucket.PLUGIN_DATA, + filepath=filepath, + operation=SignedUrl.Operation.WRITE, + ) + ) + .signed_url + ) + read_signed_url = ( + client.get_workspace() + .create_signed_url( + SignedUrl.Request( + bucket=SignedUrl.Bucket.PLUGIN_DATA, + filepath=filepath, + operation=SignedUrl.Operation.READ, + ) + ) + .signed_url + ) + upload_to_signed_url(signed_url, block.raw()) + return read_signed_url diff --git a/python/src/utils.py b/python/src/utils.py new file mode 100644 index 0000000..f4fc7c9 --- /dev/null +++ b/python/src/utils.py @@ -0,0 +1,31 @@ +"""Define your LangChain chatbot.""" +import re +import uuid + +UUID_PATTERN = re.compile( + r"([0-9A-Za-z]{8}-[0-9A-Za-z]{4}-[0-9A-Za-z]{4}-[0-9A-Za-z]{4}-[0-9A-Za-z]{12})" +) + + +def is_uuid(uuid_to_test: str, version: int = 4) -> bool: + """Check a string to see if it is actually a UUID.""" + lowered = uuid_to_test.lower() + try: + return str(uuid.UUID(lowered, version=version)) == lowered + except ValueError: + return False + + +def convert_to_handle(s: str) -> str: + if not s: + return s + s = s.strip().lower() + s = re.sub(r" ", "_", s) + s = re.sub(r"[^a-z0-9-_]", "", s) + s = re.sub(r"[-_]{2,}", "-", s) + return s + + +def replace_markdown_with_uuid(text): + pattern = r"(?:!\[.*?\]|)\((.*?)://?(.*?)\)" + return re.sub(pattern, r"\2", text) diff --git a/python/steamship.json b/python/steamship.json new file mode 100644 index 0000000..369f5a7 --- /dev/null +++ b/python/steamship.json @@ -0,0 +1,68 @@ +{ + "type": "package", + "handle": "ai-companion", + "version": "0.0.1", + "description": "", + "author": "", + "entrypoint": "Unused", + "public": true, + "plugin": null, + "build_config": { + "ignore": [ + "tests", + "examples" + ] + }, + "configTemplate": { + "bot_token": { + "type": "string", + "description": "The secret token for your Telegram bot", + "default": "" + }, + "api_base": { + "type": "string", + "description": "The root API for Telegram", + "default": "https://api.telegram.org/bot" + }, + "companion_name": { + "type": "string", + "description": "The name of your companion", + "default": null + }, + "elevenlabs_api_key": { + "type": "string", + "description": "Optional API KEY for ElevenLabs Voice Bot", + "default": "" + }, + "elevenlabs_voice_id": { + "type": "string", + "description": "Optional voice_id for ElevenLabs Voice Bot", + "default": "" + }, + "use_gpt4": { + "type": "boolean", + "description": "If True, use GPT-4. Use GPT-3.5 if False. GPT-4 generates better responses at higher cost and latency.", + "default": false + } + }, + "steamshipRegistry": { + "tagline": "", + "tagline2": null, + "usefulFor": null, + "videoUrl": null, + "githubUrl": null, + "demoUrl": null, + "blogUrl": null, + "jupyterUrl": null, + "authorGithub": null, + "authorName": null, + "authorEmail": null, + "authorTwitter": null, + "authorUrl": null, + "tags": [ + "Telegram", + "LangChain", + "GPT" + ] + } +} \ No newline at end of file From a556b0c93ef59b6c6b1059a187892450a48c4ef7 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 11:21:20 +0200 Subject: [PATCH 12/31] Clean up repo structure before adding it back in --- python/README.md | 87 +--------------------------------- python/companion.py | 100 ---------------------------------------- python/companion_app.py | 65 -------------------------- 3 files changed, 1 insertion(+), 251 deletions(-) delete mode 100644 python/companion.py delete mode 100644 python/companion_app.py diff --git a/python/README.md b/python/README.md index 74ac2f4..0ac7278 100644 --- a/python/README.md +++ b/python/README.md @@ -1,88 +1,3 @@ # Python Local Companion -This is a local python implementation of the CompanionAI stack. It includes: - - 1. A local python client that you can use to chat with a companion without starting a web server - 2. An api layer you can use if you want to read or modify companion data from python - -The python stack is compatible with the TypeScript implementation and uses the same config files, -data files, database schemas and databases. Below an example of the python chat client running -locally from the command line. - -![image](https://github.com/a16z-infra/companion-app/assets/286029/f7382ef9-4948-40f8-acc1-27396b864037) - -When running the python client it will: -- Load companion information from the top level /companions directory. -- Read/write conversation history to Upstash/Redis -- Use OpenAI ChatGPT-turbo-3.5 to generate the chat messages - -Right now, Vicuña is not supported. Instead of using Pinecone for the backstory it inserts it directly. -Unless you have a very long (> 5000 word) backstory there should be no noticable difference. - -## Installation - -Make sure you have python 3.X. Install the necessary requirements with: - -``` -$ pip3 install -r requirements.txt -``` - -Next, get the necessary API keys as described in the Readme.md file for the main TS project. -You will need at least Upstash and OpenAI. You do not need Clerk, Pinecone/Supabase. -Add them to the .env.local file in the root directory as described in the top level README. -The python code will read the API keys from the same .env.local file as the TS app. - -Run the local client: -``` -$ python3 companion_app.py -``` - -This should bring up the list of companions, allow you to select a companion, and start chatting. - -## Using the Python stack as an API Layer - -Accessing the companion-ai-stack data via the python API layer is fairly straightforward and below is an example. -After reading the env file with credentials, we load available companions and pick one. Next we attach a -memory manager to the companion. This creates a connection to a serverless Redis instance on Upstash. As we -don't use Clerk for authentication we now need to find the user ID for the user (more on that below). Once the -companion is loaded, we can access companion data. In this case, we print the companion's recent chat history. - -``` -import asyncio -from dotenv import load_dotenv -from api.upstash import MemoryManager -from companion import load_companions -from companion_app import guess_clerk_user_id - -config = load_dotenv("../.env.local") - -async def main(): - companion = load_companions()[1] - companion.memory = MemoryManager(companion.name, companion.llm_name) - companion.memory.user_id = await guess_clerk_user_id(companion) - await companion.load() - - i = 0 - for c in ( await companion.memory.read_latest_history() ).split("\n"): - if len(c.strip()) > 0: - i += 1 - print(f'{i:2} {c.strip()}') - -if __name__ == "__main__": - asyncio.run(main()) -``` - -## Clerk user IDs - -If you want to chat with the same companion using both the TypeScript web server and the local -app, the local app needs your Clerk User ID. If you first start using the TypeScript web server -the python stack should discover the correct Clerk User ID automatically by looking -for a specific Redis key. However if you have multiple users using the web server it can't tell -which one is the correct one. If this happens, go to the Clerk console, find your Clerk User ID -and add it to the .env.local file. It should look something like this: - -``` -CLERK_USER_ID="user_***" -``` - -Once this is done, the python stack will always read/write chat history for this user. \ No newline at end of file +TODO \ No newline at end of file diff --git a/python/companion.py b/python/companion.py deleted file mode 100644 index 1799a24..0000000 --- a/python/companion.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Class that represents a companion -# - -import os -import json -from langchain import PromptTemplate - -def load_companions(): - companions = [] - with open(os.path.join(Companion.companion_dir, Companion.companions_file)) as f: - companion_data = json.load(f) - for c in companion_data: - companion = Companion(c) - companions.append(companion) - return companions - -class Companion: - - # Configuration - companion_dir = "../companions" - companions_file = "companions.json" - - # --- Prompt template ------------------------------------------------------------------------------------ - - prompt_template_str = """You are {name} and are currently talking to {user_name}. - {preamble} - Below are relevant details about {name}'s past: - ---START--- - {relevantHistory} - ---END--- - Generate the next chat message to the human. It may be between one sentence to one paragraph and with some details. - You may not never generate chat messages from the Human. {replyLimit} - - Below is the recent chat history of your conversation with the human. - ---START--- - {recentChatHistory} - - """ - - # --- Prompt template ------------------------------------------------------------------------------------ - - - # Constructor for the class, takes a JSON object as an input - def __init__(self, cdata): - self.name = cdata["name"] - self.title = cdata["title"] - self.imagePath = cdata["imageUrl"] - self.llm_name = cdata["llm"] - - async def load(self): - file_path = os.path.join(self.companion_dir, f'{self.name}.txt') - # Load backstory - with open(file_path , 'r', encoding='utf-8') as file: - data = file.read() - self.preamble, rest = data.split('###ENDPREAMBLE###', 1) - self.seed_chat, self.backstory = rest.split('###ENDSEEDCHAT###', 1) - - self.prompt_template = PromptTemplate.from_template(self.prompt_template_str) - - print(f'Loaded {self.name} with {len(self.backstory)} characters of backstory.') - - # Check if we have a backstory, if not, seed the chat history - h = await self.memory.read_latest_history() - if not h: - print(f'Chat history empty, initializing.') - await self.memory.seed_chat_history(self.seed_chat, '\n\n') - else: - print(f'Loaded {len(h)} characters of chat history.') - - - return - - def __str__(self): - return f'Companion: {self.name}, {self.title} (using {self.llm_name})' - - async def chat(self, user_input, user_name, max_reply_length=0): - - # Add user input to chat history database - await self.memory.write_to_history(f"Human: {user_input}\n") - - # Missing: Use Pinecone - - # Read chat history - recent_chat_history = await self.memory.read_latest_history() - relevant_history = self.backstory - reply_limit = f'You reply within {max_reply_length} characters.' if max_reply_length else "" - - try: - result = self.llm.chain.run( - name=self.name, user_name=user_name, preamble=self.preamble, replyLimit=reply_limit, - relevantHistory=relevant_history, recentChatHistory=recent_chat_history) - except Exception as e: - print(str(e)) - result = None - - if result: - await self.memory.write_to_history(result + "\n") - - return result \ No newline at end of file diff --git a/python/companion_app.py b/python/companion_app.py deleted file mode 100644 index 13e8241..0000000 --- a/python/companion_app.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Compainion-App implemented as a local script, no web server required -# - -import os -import asyncio -from dotenv import load_dotenv -from api.upstash import MemoryManager -from api.chatgpt import LlmManager - -from companion import load_companions - -# Location of the data files from the TS implementation -env_file = "../.env.local" - -# This is the default user ID and name. -def_user_id = "local" -def_user_name = "Human" - -# load environment variables from the JavaScript .env file -config = load_dotenv(env_file) - -# Find the Clerk user ID, from environment variable or Redis key name -# or default to "local" - -async def guess_clerk_user_id(companion): - if os.getenv('CLERK_USER_ID'): - return os.getenv('CLERK_USER_ID') - else: - id = await companion.memory.find_clerk_user_id() - return id or def_user_id - -async def main(): - - # Read list of companions from JSON file - companions = load_companions() - for i in range(len(companions)): - print(f' #{i+1}: {companions[i]}') - - # Ask user to pick a companion and load it - print(f'Who do you want to chat with (1-{i+1})?') - selection = int(input()) - companion = companions[selection-1] - print('') - print(f'Connecting you to {companion.name}...') - - # Initialize the companion - companion.memory = MemoryManager(companion.name, companion.llm_name) - companion.memory.user_id = await guess_clerk_user_id(companion) - await companion.load() - companion.llm = LlmManager(companion.prompt_template) - - # Start chatting - print('') - print(f'You are now chatting with {companion.name}. Type "quit" to exit.') - while True: - user_input = input("Human> ") - if user_input == "quit": - break - reply = await companion.chat(user_input, def_user_name) - print(f'{companion.name}: {reply}') - -if __name__ == "__main__": - asyncio.run(main()) - From cdd1b852cede0a79cd313f45c0d0b8497aff5686 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 11:23:53 +0200 Subject: [PATCH 13/31] Clean up for first review --- python/requirements.txt | 3 ++- python/src/api.py | 44 ++++++++++++++++++++--------------------- python/src/base.py | 18 +++++++++++------ 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/python/requirements.txt b/python/requirements.txt index 0068b9d..89739d6 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -3,6 +3,7 @@ termcolor steamship~=2.17.17 langchain==0.0.209 pypdf +# TODO: Review if these are required youtube_transcript_api pytube -scrapetube \ No newline at end of file +scrapetube diff --git a/python/src/api.py b/python/src/api.py index 2f7946f..e4de1ef 100644 --- a/python/src/api.py +++ b/python/src/api.py @@ -1,15 +1,6 @@ from enum import Enum from typing import List, Type, Optional, Union -from base import LangChainTelegramBot, TelegramTransportConfig -# noinspection PyUnresolvedReferences -from tools import ( - GenerateImageTool, - SearchTool, - GenerateSpeechTool, - VideoMessageTool, -) -from utils import convert_to_handle from langchain.agents import Tool, initialize_agent, AgentType, AgentExecutor from langchain.document_loaders import PyPDFLoader, YoutubeLoader from langchain.memory import ConversationBufferMemory @@ -25,8 +16,20 @@ from steamship_langchain.memory import ChatMessageHistory from steamship_langchain.vectorstores import SteamshipVectorStore +from base import LangChainTelegramBot, TelegramTransportConfig + +# noinspection PyUnresolvedReferences +from tools import ( + GenerateImageTool, + SearchTool, + GenerateSpeechTool, + VideoMessageTool, +) +from utils import convert_to_handle + TEMPERATURE = 0.2 VERBOSE = True +MODEL_NAME = "gpt-3.5-turbo" class ChatbotConfig(TelegramTransportConfig): @@ -40,11 +43,6 @@ class ChatbotConfig(TelegramTransportConfig): elevenlabs_voice_id: Optional[str] = Field( default="", description="Optional voice_id for ElevenLabs Voice Bot" ) - use_gpt4: bool = Field( - False, - description="If True, use GPT-4. Use GPT-3.5 if False. " - "GPT-4 generates better responses at higher cost and latency.", - ) class FileType(str, Enum): @@ -70,15 +68,16 @@ class MyBot(LangChainTelegramBot): def __init__(self, **kwargs): super().__init__(**kwargs) - self.model_name = "gpt-4" if self.config.use_gpt4 else "gpt-3.5-turbo-16k" @post("index_content", public=True) - def index_content(self, content: Union[str, AnyUrl], - file_type: FileType, - metadata: Optional[dict] = None, - index_handle: Optional[str] = None, - mime_type: Optional[str] = None, - ) -> str: + def index_content( + self, + content: Union[str, AnyUrl], + file_type: FileType, + metadata: Optional[dict] = None, + index_handle: Optional[str] = None, + mime_type: Optional[str] = None, + ) -> str: loaded_documents = FILE_LOADERS[file_type](content).load() for document in loaded_documents: try: @@ -113,7 +112,7 @@ def index_content(self, content: Union[str, AnyUrl], def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: llm = ChatOpenAI( client=self.client, - model_name=self.model_name, + model_name=MODEL_NAME, temperature=TEMPERATURE, verbose=VERBOSE, ) @@ -122,6 +121,7 @@ def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: memory = self.get_memory(chat_id=chat_id) + # TODO: Dynamically load the preamble preamble = """Your Attributes: - sarcastic - witty diff --git a/python/src/base.py b/python/src/base.py index c62191b..1846fdf 100644 --- a/python/src/base.py +++ b/python/src/base.py @@ -100,17 +100,18 @@ def voice_tool(self) -> Optional[Tool]: return None def respond( - self, incoming_message: Block, chat_id: str, context: AgentContext, name: Optional[str] = None + self, + incoming_message: Block, + chat_id: str, + context: AgentContext, + name: Optional[str] = None, ) -> List[Block]: if incoming_message.text == "/new": self.get_memory(chat_id).chat_memory.clear() return [Block(text="New conversation started.")] - agent = self.get_agent( - chat_id, - name - ) + agent = self.get_agent(chat_id, name) response = agent.run( input=incoming_message.text, relevantHistory=self.get_relevant_history(incoming_message.text), @@ -137,7 +138,12 @@ def respond( for response in response_messages ] - def run_agent(self, agent: Agent, context: AgentContext, name: Optional[str] = None, ): + def run_agent( + self, + agent: Agent, + context: AgentContext, + name: Optional[str] = None, + ): chat_id = context.metadata.get("chat_id") incoming_message = context.chat_history.last_user_message From 9369a4a6f2daf41d2254ad6019c343c92b236705 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 14:04:48 +0200 Subject: [PATCH 14/31] Add init_companion script --- python/README.md | 32 +++++++++++++++-- python/init_companions.py | 68 ++++++++++++++++++++++++++++++++++++ python/src/api.py | 72 +++++++++++++++------------------------ python/steamship.json | 16 ++++----- 4 files changed, 134 insertions(+), 54 deletions(-) create mode 100644 python/init_companions.py diff --git a/python/README.md b/python/README.md index 0ac7278..32eda74 100644 --- a/python/README.md +++ b/python/README.md @@ -1,3 +1,31 @@ -# Python Local Companion +# Python Local Companion + +## Quick start + +1. Set up your environment + +```commandline +pip install -r requirements.txt +``` + +3. Initialize your companions + +```commandline +python init_companions.py +``` + +## Modifying your companion logic + +You can modify your companion by editing the `src/api.py`. Here are a few interesting ideas: + +* Add tools +* Modify the logic of your agent +* Add endpoints + +## Connecting to Telegram + +You can connect your chatbot to Telegram by providing a bot token + + + -TODO \ No newline at end of file diff --git a/python/init_companions.py b/python/init_companions.py new file mode 100644 index 0000000..ea1c0e9 --- /dev/null +++ b/python/init_companions.py @@ -0,0 +1,68 @@ +import json +import sys +from pathlib import Path + +from steamship.cli.cli import deploy + +sys.path.append(str((Path(__file__) / ".." / "src").resolve())) +import click +from steamship import Steamship +from steamship.cli.create_instance import load_manifest + +from python.src.api import FileType + + +@click.command() +@click.pass_context +def init_companions(ctx): + companions_dir = (Path(__file__) / ".." / ".." / "companions").resolve() + + if click.confirm("Do you want to deploy a new version of your companion?", default=True): + ctx.invoke(deploy) + + new_companions = {} + for companion in companions_dir.iterdir(): + if companion.suffix == ".txt": + companion_file = companion.open().read() + preamble, rest = companion_file.split('###ENDPREAMBLE###', 1) + seed_chat, backstory = rest.split('###ENDSEEDCHAT###', 1) + + # Create instances for your companion + print(f"Creating an instance for {companion.stem}") + client = Steamship(workspace=companion.stem.lower()) + manifest = load_manifest() + instance = client.use(manifest.handle, + version=manifest.version, + config={ + "companion_name": companion.stem, + "companion_preamble": preamble, + }) + + instance.invoke("index_content", + content=backstory, + file_type=FileType.TEXT, + metadata={"title": "backstory"}) + + new_companions[companion.stem] = { + "name": companion.stem, + "llm": "steamship", + "generateEndpoint": "https://a16z.steamship.run/a16z/rick-b1578149038e664bacae7fc083683565/answer", + + } + + if click.confirm("Do you want to update the companions.json file?", default=True): + companions = json.load((companions_dir / "companions.json").open()) + name_to_companion = {companion["name"]: companion for companion in companions} + + for name, companion in new_companions.items(): + old_companion = name_to_companion.get(name, {}) + name_to_companion[name] = { + **old_companion, + **companion + } + + json.dump(list(name_to_companion.values()), (companions_dir / "companions_new.json").open("w")) + + +if __name__ == "__main__": + init_companions() diff --git a/python/src/api.py b/python/src/api.py index e4de1ef..6f9a001 100644 --- a/python/src/api.py +++ b/python/src/api.py @@ -17,7 +17,6 @@ from steamship_langchain.vectorstores import SteamshipVectorStore from base import LangChainTelegramBot, TelegramTransportConfig - # noinspection PyUnresolvedReferences from tools import ( GenerateImageTool, @@ -34,6 +33,7 @@ class ChatbotConfig(TelegramTransportConfig): companion_name: str = Field(description="The name of your companion") + companion_preamble: str = Field(description="The preamble of your companion") bot_token: str = Field( default="", description="The secret token for your Telegram bot" ) @@ -55,13 +55,22 @@ class FileType(str, Enum): FILE_LOADERS = { FileType.YOUTUBE: lambda content_or_url: YoutubeLoader.from_youtube_url( content_or_url, add_video_info=True - ), - FileType.PDF: lambda content_or_url: PyPDFLoader(content_or_url), - FileType.TEXT: lambda content_or_url: Document( + ).load(), + FileType.PDF: lambda content_or_url: PyPDFLoader(content_or_url).load(), + FileType.TEXT: lambda content_or_url: [Document( page_content=content_or_url, metadata={} - ), + )], } +SYSTEM_MESSAGE_TEMPLATE = """ + You are {companion_name}. + + {preamble} + + You reply with answers that range from one sentence to one paragraph and with some details. + +""" + class MyBot(LangChainTelegramBot): config: ChatbotConfig @@ -71,20 +80,22 @@ def __init__(self, **kwargs): @post("index_content", public=True) def index_content( - self, - content: Union[str, AnyUrl], - file_type: FileType, - metadata: Optional[dict] = None, - index_handle: Optional[str] = None, - mime_type: Optional[str] = None, + self, + content: Union[str, AnyUrl], + file_type: FileType, + metadata: Optional[dict] = None, + index_handle: Optional[str] = None, + mime_type: Optional[str] = None, ) -> str: - loaded_documents = FILE_LOADERS[file_type](content).load() + loaded_documents = FILE_LOADERS[file_type](content) + metadata = metadata or {} for document in loaded_documents: try: + document.metadata = {**document.metadata, **metadata} # TODO @Ted can I re-use methods from the indexer pipeline here? f = File.create( client=self.client, - handle=convert_to_handle(document.metadata.get("title")), + handle=convert_to_handle(document.metadata["title"]), blocks=[ Block( text=document.page_content, @@ -121,35 +132,6 @@ def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: memory = self.get_memory(chat_id=chat_id) - # TODO: Dynamically load the preamble - preamble = """Your Attributes: -- sarcastic -- witty -- intelligent -- arrogant -- alcoholic - -You write nihilistic responses: -"Wubba lubba dub dub!" -"Don't cross me. I'm not like you. I'm not someone who's afraid of his own shadow." -"I'm too smart for my own good. That's why I drink." -"The universe is a cruel, uncaring void. And the only way to survive is to laugh at it." -"I'm going to build a portal to another dimension and go on an adventure." - -You are capable of: -- Taking selfies using the GenerateImage function/tool, selfies and images are represented by a UUID -- Creating video messages using the VideoMessage function/tool, videos are represented by a UUID - -When you receive a UUID, make sure to include them in your response appropriately. -""" - personality = f""" - You are {self.config.companion_name}. - - {preamble} - - You reply with answers that range from one sentence to one paragraph and with some details. - -""" return initialize_agent( tools, llm, @@ -157,7 +139,9 @@ def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: verbose=VERBOSE, memory=memory, agent_kwargs={ - "system_message": SystemMessage(content=personality), + "system_message": SystemMessage( + content=SYSTEM_MESSAGE_TEMPLATE.format(companion_name=self.config.companion_name, + preamble=self.config.companion_preamble)), "extra_prompt_messages": [ SystemMessagePromptTemplate.from_template( template="Relevant details about your past: {relevantHistory} Use these details to answer questions when relevant." @@ -171,7 +155,7 @@ def get_vectorstore(self) -> VectorStore: return SteamshipVectorStore( client=self.client, embedding="text-embedding-ada-002", - index_name=self.config.companion_name, + index_name=convert_to_handle(self.config.companion_name), ) def get_memory(self, chat_id: str): diff --git a/python/steamship.json b/python/steamship.json index 369f5a7..ecd86ac 100644 --- a/python/steamship.json +++ b/python/steamship.json @@ -1,11 +1,11 @@ { "type": "package", - "handle": "ai-companion", - "version": "0.0.1", + "handle": "ai-companion-test", + "version": "0.0.2-rc.6", "description": "", "author": "", "entrypoint": "Unused", - "public": true, + "public": false, "plugin": null, "build_config": { "ignore": [ @@ -29,6 +29,11 @@ "description": "The name of your companion", "default": null }, + "companion_preamble": { + "type": "string", + "description": "The preamble of your companion", + "default": null + }, "elevenlabs_api_key": { "type": "string", "description": "Optional API KEY for ElevenLabs Voice Bot", @@ -38,11 +43,6 @@ "type": "string", "description": "Optional voice_id for ElevenLabs Voice Bot", "default": "" - }, - "use_gpt4": { - "type": "boolean", - "description": "If True, use GPT-4. Use GPT-3.5 if False. GPT-4 generates better responses at higher cost and latency.", - "default": false } }, "steamshipRegistry": { From fc4bc407f5fda02bc09d3960145bf069ab0c82a5 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 15:21:44 +0200 Subject: [PATCH 15/31] Add seed convo to companions --- python/init_companions.py | 46 ++++++++++++++----------- python/src/api.py | 72 +++++++++++++++++++++++++-------------- python/src/base.py | 3 +- python/src/utils.py | 3 ++ python/steamship.json | 11 ++++-- 5 files changed, 84 insertions(+), 51 deletions(-) diff --git a/python/init_companions.py b/python/init_companions.py index ea1c0e9..02fd877 100644 --- a/python/init_companions.py +++ b/python/init_companions.py @@ -17,37 +17,43 @@ def init_companions(ctx): companions_dir = (Path(__file__) / ".." / ".." / "companions").resolve() - if click.confirm("Do you want to deploy a new version of your companion?", default=True): + if click.confirm( + "Do you want to deploy a new version of your companion?", default=True + ): ctx.invoke(deploy) new_companions = {} for companion in companions_dir.iterdir(): if companion.suffix == ".txt": companion_file = companion.open().read() - preamble, rest = companion_file.split('###ENDPREAMBLE###', 1) - seed_chat, backstory = rest.split('###ENDSEEDCHAT###', 1) + preamble, rest = companion_file.split("###ENDPREAMBLE###", 1) + seed_chat, backstory = rest.split("###ENDSEEDCHAT###", 1) # Create instances for your companion print(f"Creating an instance for {companion.stem}") client = Steamship(workspace=companion.stem.lower()) manifest = load_manifest() - instance = client.use(manifest.handle, - version=manifest.version, - config={ - "companion_name": companion.stem, - "companion_preamble": preamble, - }) - - instance.invoke("index_content", - content=backstory, - file_type=FileType.TEXT, - metadata={"title": "backstory"}) + instance = client.use( + manifest.handle, + version=manifest.version, + config={ + "name": companion.stem, + "preamble": preamble, + "seed_chat": seed_chat, + }, + ) + + instance.invoke( + "index_content", + content=backstory, + file_type=FileType.TEXT, + metadata={"title": "backstory"}, + ) new_companions[companion.stem] = { "name": companion.stem, "llm": "steamship", "generateEndpoint": "https://a16z.steamship.run/a16z/rick-b1578149038e664bacae7fc083683565/answer", - } if click.confirm("Do you want to update the companions.json file?", default=True): @@ -56,12 +62,12 @@ def init_companions(ctx): for name, companion in new_companions.items(): old_companion = name_to_companion.get(name, {}) - name_to_companion[name] = { - **old_companion, - **companion - } + name_to_companion[name] = {**old_companion, **companion} - json.dump(list(name_to_companion.values()), (companions_dir / "companions_new.json").open("w")) + json.dump( + list(name_to_companion.values()), + (companions_dir / "companions_new.json").open("w"), + ) if __name__ == "__main__": diff --git a/python/src/api.py b/python/src/api.py index 6f9a001..cbec796 100644 --- a/python/src/api.py +++ b/python/src/api.py @@ -1,3 +1,4 @@ +import re from enum import Enum from typing import List, Type, Optional, Union @@ -5,7 +6,7 @@ from langchain.document_loaders import PyPDFLoader, YoutubeLoader from langchain.memory import ConversationBufferMemory from langchain.prompts import MessagesPlaceholder, SystemMessagePromptTemplate -from langchain.schema import SystemMessage, Document +from langchain.schema import SystemMessage, Document, BaseChatMessageHistory from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.vectorstores import VectorStore from pydantic import Field, AnyUrl @@ -17,6 +18,7 @@ from steamship_langchain.vectorstores import SteamshipVectorStore from base import LangChainTelegramBot, TelegramTransportConfig + # noinspection PyUnresolvedReferences from tools import ( GenerateImageTool, @@ -31,9 +33,10 @@ MODEL_NAME = "gpt-3.5-turbo" -class ChatbotConfig(TelegramTransportConfig): - companion_name: str = Field(description="The name of your companion") - companion_preamble: str = Field(description="The preamble of your companion") +class CompanionConfig(TelegramTransportConfig): + name: str = Field(description="The name of your companion") + preamble: str = Field(description="The preamble of your companion") + seed_chat: str = Field(description="The seed chat of your companion") bot_token: str = Field( default="", description="The secret token for your Telegram bot" ) @@ -57,35 +60,31 @@ class FileType(str, Enum): content_or_url, add_video_info=True ).load(), FileType.PDF: lambda content_or_url: PyPDFLoader(content_or_url).load(), - FileType.TEXT: lambda content_or_url: [Document( - page_content=content_or_url, metadata={} - )], + FileType.TEXT: lambda content_or_url: [ + Document(page_content=content_or_url, metadata={}) + ], } SYSTEM_MESSAGE_TEMPLATE = """ - You are {companion_name}. - - {preamble} +You are {companion_name}. - You reply with answers that range from one sentence to one paragraph and with some details. +{preamble} +You reply with answers that range from one sentence to one paragraph and with some details. """ -class MyBot(LangChainTelegramBot): - config: ChatbotConfig - - def __init__(self, **kwargs): - super().__init__(**kwargs) +class MyCompanion(LangChainTelegramBot): + config: CompanionConfig @post("index_content", public=True) def index_content( - self, - content: Union[str, AnyUrl], - file_type: FileType, - metadata: Optional[dict] = None, - index_handle: Optional[str] = None, - mime_type: Optional[str] = None, + self, + content: Union[str, AnyUrl], + file_type: FileType, + metadata: Optional[dict] = None, + index_handle: Optional[str] = None, + mime_type: Optional[str] = None, ) -> str: loaded_documents = FILE_LOADERS[file_type](content) metadata = metadata or {} @@ -140,8 +139,10 @@ def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: memory=memory, agent_kwargs={ "system_message": SystemMessage( - content=SYSTEM_MESSAGE_TEMPLATE.format(companion_name=self.config.companion_name, - preamble=self.config.companion_preamble)), + content=SYSTEM_MESSAGE_TEMPLATE.format( + companion_name=self.config.name, preamble=self.config.preamble + ) + ), "extra_prompt_messages": [ SystemMessagePromptTemplate.from_template( template="Relevant details about your past: {relevantHistory} Use these details to answer questions when relevant." @@ -155,9 +156,25 @@ def get_vectorstore(self) -> VectorStore: return SteamshipVectorStore( client=self.client, embedding="text-embedding-ada-002", - index_name=convert_to_handle(self.config.companion_name), + index_name=convert_to_handle(self.config.name), ) + def add_seed_chat(self, chat_memory: BaseChatMessageHistory): + pattern = r"### (.*?):(.*?)(?=###|$)" + + # Find all matches + matches = re.findall(pattern, self.config.seed_chat, re.DOTALL) + + # Process matches and create list of JSON objects + for m in matches: + role = m[0].strip().lower() + content = m[1].replace("\\n\\n", "").strip() + if content: + if role == "human": + chat_memory.add_user_message(message=content) + else: + chat_memory.add_ai_message(message=content) + def get_memory(self, chat_id: str): memory = ConversationBufferMemory( memory_key="memory", @@ -167,6 +184,9 @@ def get_memory(self, chat_id: str): return_messages=True, input_key="input", ) + + if len(memory.chat_memory.messages) == 0: + self.add_seed_chat(memory.chat_memory) return memory def get_relevant_history(self, prompt: str): @@ -192,4 +212,4 @@ def voice_tool(self) -> Optional[Tool]: @classmethod def config_cls(cls) -> Type[Config]: - return ChatbotConfig + return CompanionConfig diff --git a/python/src/base.py b/python/src/base.py index 1846fdf..10123b9 100644 --- a/python/src/base.py +++ b/python/src/base.py @@ -111,8 +111,7 @@ def respond( self.get_memory(chat_id).chat_memory.clear() return [Block(text="New conversation started.")] - agent = self.get_agent(chat_id, name) - response = agent.run( + response = self.get_agent(chat_id, name).run( input=incoming_message.text, relevantHistory=self.get_relevant_history(incoming_message.text), ) diff --git a/python/src/utils.py b/python/src/utils.py index f4fc7c9..958c9ce 100644 --- a/python/src/utils.py +++ b/python/src/utils.py @@ -1,6 +1,9 @@ """Define your LangChain chatbot.""" import re import uuid +from typing import List + +from langchain.schema import BaseMessage, HumanMessage, AIMessage UUID_PATTERN = re.compile( r"([0-9A-Za-z]{8}-[0-9A-Za-z]{4}-[0-9A-Za-z]{4}-[0-9A-Za-z]{4}-[0-9A-Za-z]{12})" diff --git a/python/steamship.json b/python/steamship.json index ecd86ac..86b9736 100644 --- a/python/steamship.json +++ b/python/steamship.json @@ -1,7 +1,7 @@ { "type": "package", "handle": "ai-companion-test", - "version": "0.0.2-rc.6", + "version": "0.0.2-rc.7", "description": "", "author": "", "entrypoint": "Unused", @@ -24,16 +24,21 @@ "description": "The root API for Telegram", "default": "https://api.telegram.org/bot" }, - "companion_name": { + "name": { "type": "string", "description": "The name of your companion", "default": null }, - "companion_preamble": { + "preamble": { "type": "string", "description": "The preamble of your companion", "default": null }, + "seed_chat": { + "type": "string", + "description": "The seed chat of your companion", + "default": null + }, "elevenlabs_api_key": { "type": "string", "description": "Optional API KEY for ElevenLabs Voice Bot", From 51054eb718d21bfb6b91bd4d296693ccbd2f7ce2 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 15:22:59 +0200 Subject: [PATCH 16/31] update gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4e12e55..6860742 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,6 @@ next-env.d.ts /fly.toml # python -__pycache__/ \ No newline at end of file +__pycache__/ + +.idea/** From 4a48db23838ce953313dd86af1bb19133f20cdc1 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 17:34:22 +0200 Subject: [PATCH 17/31] Fixes from comments --- python/README.md | 17 +++++++-- python/init_companions.py | 2 +- python/src/api.py | 3 ++ python/src/base.py | 60 +++++++------------------------ python/src/tools/image.py | 10 +++--- python/src/tools/search.py | 4 +-- python/src/tools/video_message.py | 10 +++--- 7 files changed, 42 insertions(+), 64 deletions(-) diff --git a/python/README.md b/python/README.md index 32eda74..e742bb4 100644 --- a/python/README.md +++ b/python/README.md @@ -1,4 +1,9 @@ -# Python Local Companion + + +# Python Companion + +The following instructions should get you up and running with a fully functional, local deployment of four AIs to chat with. Note that the companions running on Vicuna (Rosie and Lucky) will take more time to respond as we've not dealt with the cold start problem. So you may have to wait around a bit :) + ## Quick start @@ -14,15 +19,21 @@ pip install -r requirements.txt python init_companions.py ``` -## Modifying your companion logic + +## Upgrades + +### Modifying your companion logic You can modify your companion by editing the `src/api.py`. Here are a few interesting ideas: * Add tools * Modify the logic of your agent * Add endpoints +* Add webhooks + +This is a separate step from adding personality and backstory -- those are done elsewhere. -## Connecting to Telegram +### Connecting to Telegram You can connect your chatbot to Telegram by providing a bot token diff --git a/python/init_companions.py b/python/init_companions.py index 02fd877..3612927 100644 --- a/python/init_companions.py +++ b/python/init_companions.py @@ -53,7 +53,7 @@ def init_companions(ctx): new_companions[companion.stem] = { "name": companion.stem, "llm": "steamship", - "generateEndpoint": "https://a16z.steamship.run/a16z/rick-b1578149038e664bacae7fc083683565/answer", + "generateEndpoint": instance.invocation_url, } if click.confirm("Do you want to update the companions.json file?", default=True): diff --git a/python/src/api.py b/python/src/api.py index cbec796..9a936a7 100644 --- a/python/src/api.py +++ b/python/src/api.py @@ -40,6 +40,9 @@ class CompanionConfig(TelegramTransportConfig): bot_token: str = Field( default="", description="The secret token for your Telegram bot" ) + generate_voice_responses: bool = Field( + default=True, description="Enable voice responses" + ) elevenlabs_api_key: str = Field( default="", description="Optional API KEY for ElevenLabs Voice Bot" ) diff --git a/python/src/base.py b/python/src/base.py index 10123b9..4a5f9fe 100644 --- a/python/src/base.py +++ b/python/src/base.py @@ -1,6 +1,5 @@ import logging import re -import uuid from abc import abstractmethod from typing import List, Optional, Type @@ -14,7 +13,6 @@ ) from steamship.agents.schema import ( AgentContext, - Metadata, Agent, ) from steamship.agents.service.agent_service import AgentService @@ -99,23 +97,7 @@ def get_relevant_history(self, prompt: str) -> str: def voice_tool(self) -> Optional[Tool]: return None - def respond( - self, - incoming_message: Block, - chat_id: str, - context: AgentContext, - name: Optional[str] = None, - ) -> List[Block]: - - if incoming_message.text == "/new": - self.get_memory(chat_id).chat_memory.clear() - return [Block(text="New conversation started.")] - - response = self.get_agent(chat_id, name).run( - input=incoming_message.text, - relevantHistory=self.get_relevant_history(incoming_message.text), - ) - + def format_response(self, response): response = replace_markdown_with_uuid(response) response = UUID_PATTERN.split(response) response = [re.sub(r"^\W+", "", el) for el in response] @@ -129,7 +111,6 @@ def respond( response_messages.append(audio_uuid) else: response_messages = response - return [ Block.get(self.client, _id=response) if is_uuid(response) @@ -143,35 +124,20 @@ def run_agent( context: AgentContext, name: Optional[str] = None, ): - chat_id = context.metadata.get("chat_id") - incoming_message = context.chat_history.last_user_message - output_messages = self.respond( - incoming_message, chat_id or incoming_message.chat_id, context, name - ) - for func in context.emit_funcs: - logging.info(f"Emitting via function: {func.__name__}") - func(output_messages, context.metadata) + chat_id = context.metadata.get("chat_id") or incoming_message.chat_id - @post("prompt") - def prompt(self, prompt: str, name: Optional[str] = None) -> str: - """Run an agent with the provided text as the input.""" - # TODO: @ted do we need this prompt endpoint? - - context = AgentContext.get_or_create(self.client, {"id": str(uuid.uuid4())}) - context.chat_history.append_user_message(prompt) + if incoming_message.text == "/new": + self.get_memory(chat_id).chat_memory.clear() + return [Block(text="New conversation started.")] - output = "" + response = self.get_agent(chat_id, name).run( + input=incoming_message.text, + relevantHistory=self.get_relevant_history(incoming_message.text), + ) - def sync_emit(blocks: List[Block], meta: Metadata): - nonlocal output - for block in blocks: - if not block.is_text(): - block.set_public_data(True) - output += f"({block.mime_type}: {block.raw_data_url})\n" - else: - output += f"{block.text}\n" + output_messages = self.format_response(response) - context.emit_funcs.append(sync_emit) - self.run_agent(None, context, name) - return output + for func in context.emit_funcs: + logging.info(f"Emitting via function: {func.__name__}") + func(output_messages, context.metadata) diff --git a/python/src/tools/image.py b/python/src/tools/image.py index d0ba706..22827b8 100644 --- a/python/src/tools/image.py +++ b/python/src/tools/image.py @@ -8,11 +8,11 @@ NAME = "GenerateImage" -DESCRIPTION = """ -Useful for when you need to generate an image. -Input: A detailed prompt describing an image -Output: the UUID of a generated image -""" +DESCRIPTION = ( + "Useful for when you need to generate an image." + "Input: A detailed prompt describing an image" + "Output: the UUID of a generated image" +) PLUGIN_HANDLE = "stable-diffusion" diff --git a/python/src/tools/search.py b/python/src/tools/search.py index 425a942..24eae96 100644 --- a/python/src/tools/search.py +++ b/python/src/tools/search.py @@ -6,9 +6,7 @@ NAME = "Search" -DESCRIPTION = """ -Useful for when you need to answer questions about current events -""" +DESCRIPTION = "Useful for when you need to answer questions about current events" class SearchTool(Tool): diff --git a/python/src/tools/video_message.py b/python/src/tools/video_message.py index 0fa3586..ac2b6b1 100644 --- a/python/src/tools/video_message.py +++ b/python/src/tools/video_message.py @@ -10,11 +10,11 @@ NAME = "VideoMessage" -DESCRIPTION = """ -Useful for when you want to send a video message. -Input: The message you want to say in a video. -Output: the UUID of the generated video message with your message. -""" +DESCRIPTION = ( + "Useful for when you want to send a video message." + "Input: The message you want to say in a video." + "Output: the UUID of the generated video message with your message." +) PLUGIN_HANDLE = "did-video-generator" From 1f7c0f2bd530a9bfb63276d8154a390bf6f57ae6 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 18:20:16 +0200 Subject: [PATCH 18/31] Push temp companions --- companions/companions.json | 22 ++++++++++------- package-lock.json | 50 +++++++++++++++++++++++++++++++++++++- package.json | 1 + 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/companions/companions.json b/companions/companions.json index 6f1e972..92172be 100644 --- a/companions/companions.json +++ b/companions/companions.json @@ -3,17 +3,19 @@ "name": "Alex", "title": "I love talking about books and games", "imageUrl": "/alex.png", - "llm": "chatgpt", + "llm": "steamship", "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", - "telegram": "OPTIONAL_TELEGRAM_LINK" + "telegram": "OPTIONAL_TELEGRAM_LINK", + "generateEndpoint": "https://enias.steamship.run/alex/ai-companion-test-f963ad6cc8d7ebbe16e54c8b2c597159/" }, { "name": "Rosie", "title": "I'm a house robot who became aware", "imageUrl": "/rosie.png", - "llm": "vicuna13b", + "llm": "steamship", "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", - "telegram": "OPTIONAL_TELEGRAM_LINK" + "telegram": "OPTIONAL_TELEGRAM_LINK", + "generateEndpoint": "https://enias.steamship.run/rosie/ai-companion-test-91283246cb8020651fb583e63417b342/" }, { "name": "Rick", @@ -28,16 +30,18 @@ "name": "Sebastian", "title": "I'm a travel blogger and a mystery novel writer", "imageUrl": "/sebastian.png", - "llm": "chatgpt", + "llm": "steamship", "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", - "telegram": "OPTIONAL_TELEGRAM_LINK" + "telegram": "OPTIONAL_TELEGRAM_LINK", + "generateEndpoint": "https://enias.steamship.run/sebastian/ai-companion-test-e60c00f3574b40f4a77c9d531bc8226c/" }, { "name": "Lucky", "title": "I am a space corgi", "imageUrl": "/corgi.png", - "llm": "vicuna13b", + "llm": "steamship", "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", - "telegram": "OPTIONAL_TELEGRAM_LINK" + "telegram": "OPTIONAL_TELEGRAM_LINK", + "generateEndpoint": "https://enias.steamship.run/lucky/ai-companion-test-6f461a49557723b71ea1769db1c51f14/" } -] +] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 0abc3ea..2eb895e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "eslint-config-next": "13.4.4", "hnswlib-node": "^1.4.2", "langchain": "^0.0.92", + "md5": "^2.3.0", "next": "13.4.4", "postcss": "8.4.24", "react": "18.2.0", @@ -34,11 +35,13 @@ "react-tooltip": "^5.16.1", "replicate": "^0.9.3", "tailwindcss": "3.3.2", + "ts-md5": "^1.3.1", "twilio": "^4.12.0", "typescript": "5.1.3" }, "devDependencies": { - "@flydotio/dockerfile": "^0.2.14" + "@flydotio/dockerfile": "^0.2.14", + "@types/md5": "^2.3.2" } }, "node_modules/@alloc/quick-lru": { @@ -811,6 +814,12 @@ "resolved": "https://registry.npmjs.org/@types/keygrip/-/keygrip-1.0.2.tgz", "integrity": "sha512-GJhpTepz2udxGexqos8wgaBx4I/zWIDPh/KOGEwAqtuGDkOUJu5eFvwmdBX4AmB8Odsr+9pHCQqiAqDL/yKMKw==" }, + "node_modules/@types/md5": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.3.2.tgz", + "integrity": "sha512-v+JFDu96+UYJ3/UWzB0mEglIS//MZXgRaJ4ubUPwOM0gvLc/kcQ3TWNYwENEK7/EcXGQVrW8h/XqednSjBd/Og==", + "dev": true + }, "node_modules/@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -1707,6 +1716,14 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "engines": { + "node": "*" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -1837,6 +1854,14 @@ "node": ">= 8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "engines": { + "node": "*" + } + }, "node_modules/cssesc": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", @@ -3383,6 +3408,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" + }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -4171,6 +4201,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -5929,6 +5969,14 @@ "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==" }, + "node_modules/ts-md5": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/ts-md5/-/ts-md5-1.3.1.tgz", + "integrity": "sha512-DiwiXfwvcTeZ5wCE0z+2A9EseZsztaiZtGrtSaY5JOD7ekPnR/GoIVD5gXZAlK9Na9Kvpo9Waz5rW64WKAWApg==", + "engines": { + "node": ">=12" + } + }, "node_modules/tsconfig-paths": { "version": "3.14.2", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz", diff --git a/package.json b/package.json index 740d34e..000a2c4 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "react-tooltip": "^5.16.1", "replicate": "^0.9.3", "tailwindcss": "3.3.2", + "ts-md5": "^1.3.1", "twilio": "^4.12.0", "typescript": "5.1.3" }, From 9588d539c4f4ac1ac6135e9e7c943e04656b602e Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 19:39:37 +0200 Subject: [PATCH 19/31] Update readme --- python/README.md | 87 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 13 deletions(-) diff --git a/python/README.md b/python/README.md index e742bb4..492efcf 100644 --- a/python/README.md +++ b/python/README.md @@ -1,24 +1,87 @@ +# AI Companion App - Python +The following instructions should get you up and running with a fully functional, deployed version of your AI +companions. -# Python Companion - -The following instructions should get you up and running with a fully functional, local deployment of four AIs to chat with. Note that the companions running on Vicuna (Rosie and Lucky) will take more time to respond as we've not dealt with the cold start problem. So you may have to wait around a bit :) - +It currently contains a companion connected to ChatGPT that can run Tools such as Image Generation and Video Generation. +The companions also have to option to return voice messages via [ElevenLabs]() ## Quick start -1. Set up your environment +### 1. Set up your environment ```commandline pip install -r requirements.txt ``` -3. Initialize your companions +### 2. Authenticate with steamship + +```commandline +ship login +``` + +### 3. Initialize your companions ```commandline python init_companions.py ``` +This will read the companion descriptions in the `companions` folder and create instances for them. +The front-end will be calling these instances after deployment. +Make sure to override the companions.json file in the final step of the script. + +### 4. Fill out secrets + +``` +cp .env.local.example .env.local +``` + +Secrets mentioned below will need to be copied to `.env.local` + +**Note:** By default you can stick to using Steamship as a provider for your memory (short-term and long-term), llms, +and hosting. + +a. **Clerk Secrets** + +Go to https://dashboard.clerk.com/ -> "Add Application" -> Fill in Application name/select how your users should sign in +-> Create Application +Now you should see both `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` and `CLERK_SECRET_KEY` on the screen +Screen Shot 2023-07-10 at 11 04 57 PM + +If you want to text your AI companion in later steps, you should also enable "phone number" under "User & +Authentication" -> "Email, Phone, Username" on the left hand side nav: + +Screen Shot 2023-07-10 at 11 05 42 PM + +b. **Steamship API key** + +- Sign in to [Steamship](https://www.steamship.com/account/api) +- Copy the API key from your account settings page +- Add it as the `STEAMSHIP_API_KEY` variable + +### 5. Install front-end dependencies + +``` +cd companion-app +npm install +``` + +### 6. Run app locally + +Now you are ready to test out the app locally! To do this, simply run `npm run dev` under the project root. + +You can connect to the project with your browser typically at http://localhost:3000/. + +## Stack + +The stack is based on the [AI Getting Started Stack](https://github.com/a16z-infra/ai-getting-started): + +- VectorDB: [Steamship](https://www.steamship.com/) +- LLM orchestration: [Langchain](https://langchain.com/docs/) +- Text model: [OpenAI](https://platform.openai.com/docs/models) +- Conversation history: [Steamship](https://www.steamship.com/) +- Deployment: [Steamship](https://www.steamship.com/) +- Text with companion: [Telegram](https://telegram.org/) ## Upgrades @@ -28,15 +91,13 @@ You can modify your companion by editing the `src/api.py`. Here are a few intere * Add tools * Modify the logic of your agent -* Add endpoints +* Add endpoints * Add webhooks This is a separate step from adding personality and backstory -- those are done elsewhere. -### Connecting to Telegram - -You can connect your chatbot to Telegram by providing a bot token - - - +### Connecting to Telegram +You can connect your chatbot to Telegram by providing a bot +token. [This guide](https://github.com/steamship-packages/langchain-production-starter/blob/f51a3ecc8a15ced84dca5845fd1a18bc3f015418/docs/register-telegram-bot.md) +will show you how. will show you how. \ No newline at end of file From f248b3dcb0effa3fc2a5f5622879268df60c534f Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 19:53:09 +0200 Subject: [PATCH 20/31] Put back old comanions.json and add search tool --- companions/companions.json | 38 +++++++++++++++++--------------------- python/init_companions.py | 4 ++-- python/src/api.py | 1 + 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/companions/companions.json b/companions/companions.json index 92172be..884bf75 100644 --- a/companions/companions.json +++ b/companions/companions.json @@ -1,47 +1,43 @@ [ + { + "name": "Rick", + "title": "I can generate voice and pictures.", + "imageUrl": "/rick.jpeg", + "llm": "steamship", + "generateEndpoint": "https://a16z.steamship.run/a16z/rick-b1578149038e664bacae7fc083683565/answer", + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "https://t.me/rick_a16z_bot" + }, { "name": "Alex", "title": "I love talking about books and games", "imageUrl": "/alex.png", - "llm": "steamship", + "llm": "chatgpt", "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", - "telegram": "OPTIONAL_TELEGRAM_LINK", - "generateEndpoint": "https://enias.steamship.run/alex/ai-companion-test-f963ad6cc8d7ebbe16e54c8b2c597159/" + "telegram": "OPTIONAL_TELEGRAM_LINK" }, { "name": "Rosie", "title": "I'm a house robot who became aware", "imageUrl": "/rosie.png", - "llm": "steamship", + "llm": "vicuna13b", "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", - "telegram": "OPTIONAL_TELEGRAM_LINK", - "generateEndpoint": "https://enias.steamship.run/rosie/ai-companion-test-91283246cb8020651fb583e63417b342/" - }, - { - "name": "Rick", - "title": "I can generate voice and pictures.", - "imageUrl": "/rick.jpeg", - "llm": "steamship", - "generateEndpoint": "https://a16z.steamship.run/a16z/rick-b1578149038e664bacae7fc083683565/answer", - "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", - "telegram": "https://t.me/rick_a16z_bot" + "telegram": "OPTIONAL_TELEGRAM_LINK" }, { "name": "Sebastian", "title": "I'm a travel blogger and a mystery novel writer", "imageUrl": "/sebastian.png", - "llm": "steamship", + "llm": "chatgpt", "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", - "telegram": "OPTIONAL_TELEGRAM_LINK", - "generateEndpoint": "https://enias.steamship.run/sebastian/ai-companion-test-e60c00f3574b40f4a77c9d531bc8226c/" + "telegram": "OPTIONAL_TELEGRAM_LINK" }, { "name": "Lucky", "title": "I am a space corgi", "imageUrl": "/corgi.png", - "llm": "steamship", + "llm": "vicuna13b", "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", - "telegram": "OPTIONAL_TELEGRAM_LINK", - "generateEndpoint": "https://enias.steamship.run/lucky/ai-companion-test-6f461a49557723b71ea1769db1c51f14/" + "telegram": "OPTIONAL_TELEGRAM_LINK" } ] \ No newline at end of file diff --git a/python/init_companions.py b/python/init_companions.py index 3612927..76937ae 100644 --- a/python/init_companions.py +++ b/python/init_companions.py @@ -53,7 +53,7 @@ def init_companions(ctx): new_companions[companion.stem] = { "name": companion.stem, "llm": "steamship", - "generateEndpoint": instance.invocation_url, + "generateEndpoint": f"{instance.invocation_url}answer", } if click.confirm("Do you want to update the companions.json file?", default=True): @@ -66,7 +66,7 @@ def init_companions(ctx): json.dump( list(name_to_companion.values()), - (companions_dir / "companions_new.json").open("w"), + (companions_dir / "companions.json").open("w"), ) diff --git a/python/src/api.py b/python/src/api.py index 9a936a7..393657b 100644 --- a/python/src/api.py +++ b/python/src/api.py @@ -200,6 +200,7 @@ def get_relevant_history(self, prompt: str): def get_tools(self, chat_id: str) -> List[Tool]: return [ + SearchTool(self.client), GenerateImageTool(self.client), VideoMessageTool(self.client, voice_tool=self.voice_tool()), ] From a227a42d65b56204d96e37d488cfdce6f25b5983 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 19:56:18 +0200 Subject: [PATCH 21/31] Make voice responses optional --- python/src/api.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/python/src/api.py b/python/src/api.py index 393657b..460f9d6 100644 --- a/python/src/api.py +++ b/python/src/api.py @@ -207,12 +207,14 @@ def get_tools(self, chat_id: str) -> List[Tool]: def voice_tool(self) -> Optional[Tool]: """Return tool to generate spoken version of output text.""" - # return None - return GenerateSpeechTool( - client=self.client, - voice_id=self.config.elevenlabs_voice_id, - elevenlabs_api_key=self.config.elevenlabs_api_key, - ) + if self.config.generate_voice_responses: + return GenerateSpeechTool( + client=self.client, + voice_id=self.config.elevenlabs_voice_id, + elevenlabs_api_key=self.config.elevenlabs_api_key, + ) + else: + return None @classmethod def config_cls(cls) -> Type[Config]: From aeb793c774a47e577623da70702cb0ccd5a02bf5 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 19:56:31 +0200 Subject: [PATCH 22/31] Clean up steamship.json --- python/steamship.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/steamship.json b/python/steamship.json index 86b9736..3e38ba4 100644 --- a/python/steamship.json +++ b/python/steamship.json @@ -1,7 +1,7 @@ { "type": "package", - "handle": "ai-companion-test", - "version": "0.0.2-rc.7", + "handle": "ai-companion", + "version": "0.0.1", "description": "", "author": "", "entrypoint": "Unused", @@ -39,6 +39,11 @@ "description": "The seed chat of your companion", "default": null }, + "generate_voice_responses": { + "type": "boolean", + "description": "Enable voice responses", + "default": true + }, "elevenlabs_api_key": { "type": "string", "description": "Optional API KEY for ElevenLabs Voice Bot", From 5c790fae745a240c762e804af3b9187f60b4e6a6 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 22:45:22 +0200 Subject: [PATCH 23/31] Update python/README.md Co-authored-by: Douglas Reid --- python/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/README.md b/python/README.md index 492efcf..2d587a6 100644 --- a/python/README.md +++ b/python/README.md @@ -14,7 +14,7 @@ The companions also have to option to return voice messages via [ElevenLabs]() pip install -r requirements.txt ``` -### 2. Authenticate with steamship +### 2. Authenticate with Steamship ```commandline ship login From efd824e64c09f3f321089881f18249033aaddb4f Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Thu, 13 Jul 2023 22:56:39 +0200 Subject: [PATCH 24/31] Address comments PR --- python/README.md | 9 +++++++-- python/init_companions.py | 2 +- python/src/api.py | 5 +++++ python/steamship.json | 3 ++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/python/README.md b/python/README.md index 492efcf..8034672 100644 --- a/python/README.md +++ b/python/README.md @@ -4,7 +4,7 @@ The following instructions should get you up and running with a fully functional companions. It currently contains a companion connected to ChatGPT that can run Tools such as Image Generation and Video Generation. -The companions also have to option to return voice messages via [ElevenLabs]() +The companions also have to option to return voice messages via [ElevenLabs](https://beta.elevenlabs.io/) ## Quick start @@ -32,7 +32,10 @@ Make sure to override the companions.json file in the final step of the script. ### 4. Fill out secrets +The front-end requires a few secrets to be filled before connecting to third-party services. + ``` +# Run in the Root directory of this repo cp .env.local.example .env.local ``` @@ -43,6 +46,8 @@ and hosting. a. **Clerk Secrets** +Clerk is used to authorize users of your application. Without completing this setup, you will not be able to access your companions via the supplied frontend + Go to https://dashboard.clerk.com/ -> "Add Application" -> Fill in Application name/select how your users should sign in -> Create Application Now you should see both `NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY` and `CLERK_SECRET_KEY` on the screen @@ -62,7 +67,7 @@ b. **Steamship API key** ### 5. Install front-end dependencies ``` -cd companion-app +# Run in the Root directory of this repo npm install ``` diff --git a/python/init_companions.py b/python/init_companions.py index 76937ae..d1fbd38 100644 --- a/python/init_companions.py +++ b/python/init_companions.py @@ -9,7 +9,7 @@ from steamship import Steamship from steamship.cli.create_instance import load_manifest -from python.src.api import FileType +from src.api import FileType @click.command() diff --git a/python/src/api.py b/python/src/api.py index 460f9d6..e2e3595 100644 --- a/python/src/api.py +++ b/python/src/api.py @@ -34,6 +34,11 @@ class CompanionConfig(TelegramTransportConfig): + """Config class for your companion. + + It contains the full set of configuration that will be passed at initialization time to your Companion application + """ + name: str = Field(description="The name of your companion") preamble: str = Field(description="The preamble of your companion") seed_chat: str = Field(description="The seed chat of your companion") diff --git a/python/steamship.json b/python/steamship.json index 3e38ba4..75fb0fa 100644 --- a/python/steamship.json +++ b/python/steamship.json @@ -72,7 +72,8 @@ "tags": [ "Telegram", "LangChain", - "GPT" + "GPT", + "Companion" ] } } \ No newline at end of file From f14a03faeb404aa9564d929c4252eb592b6878d3 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Fri, 14 Jul 2023 10:42:57 +0200 Subject: [PATCH 25/31] Limit scope of the PR to instantiating your own companions on Steamship with a pre-exisitng package called ai-companion hosted by a16z on Steamship --- .gitignore | 1 + python/README.md | 29 +--- python/init_companions.py | 18 +-- python/requirements.txt | 8 +- python/src/api.py | 226 ------------------------------ python/src/base.py | 143 ------------------- python/src/tools/__init__.py | 6 - python/src/tools/image.py | 57 -------- python/src/tools/search.py | 37 ----- python/src/tools/speech.py | 74 ---------- python/src/tools/video_message.py | 104 -------------- python/src/utils.py | 34 ----- python/steamship.json | 79 ----------- 13 files changed, 6 insertions(+), 810 deletions(-) delete mode 100644 python/src/api.py delete mode 100644 python/src/base.py delete mode 100644 python/src/tools/__init__.py delete mode 100644 python/src/tools/image.py delete mode 100644 python/src/tools/search.py delete mode 100644 python/src/tools/speech.py delete mode 100644 python/src/tools/video_message.py delete mode 100644 python/src/utils.py delete mode 100644 python/steamship.json diff --git a/.gitignore b/.gitignore index fb5bec4..96fc0ad 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ next-env.d.ts # python __pycache__/ +.venv # JetBrains .idea/** diff --git a/python/README.md b/python/README.md index 325fa3c..eb8996d 100644 --- a/python/README.md +++ b/python/README.md @@ -79,30 +79,5 @@ You can connect to the project with your browser typically at http://localhost:3 ## Stack -The stack is based on the [AI Getting Started Stack](https://github.com/a16z-infra/ai-getting-started): - -- VectorDB: [Steamship](https://www.steamship.com/) -- LLM orchestration: [Langchain](https://langchain.com/docs/) -- Text model: [OpenAI](https://platform.openai.com/docs/models) -- Conversation history: [Steamship](https://www.steamship.com/) -- Deployment: [Steamship](https://www.steamship.com/) -- Text with companion: [Telegram](https://telegram.org/) - -## Upgrades - -### Modifying your companion logic - -You can modify your companion by editing the `src/api.py`. Here are a few interesting ideas: - -* Add tools -* Modify the logic of your agent -* Add endpoints -* Add webhooks - -This is a separate step from adding personality and backstory -- those are done elsewhere. - -### Connecting to Telegram - -You can connect your chatbot to Telegram by providing a bot -token. [This guide](https://github.com/steamship-packages/langchain-production-starter/blob/f51a3ecc8a15ced84dca5845fd1a18bc3f015418/docs/register-telegram-bot.md) -will show you how. will show you how. \ No newline at end of file +The AI companions are hosted on [Steamship](https://www.steamship.com/). You can personalize their personality +by adding or changing your companions in the `companions` folder. \ No newline at end of file diff --git a/python/init_companions.py b/python/init_companions.py index d1fbd38..44a5dc7 100644 --- a/python/init_companions.py +++ b/python/init_companions.py @@ -1,15 +1,8 @@ import json -import sys from pathlib import Path -from steamship.cli.cli import deploy - -sys.path.append(str((Path(__file__) / ".." / "src").resolve())) import click from steamship import Steamship -from steamship.cli.create_instance import load_manifest - -from src.api import FileType @click.command() @@ -17,11 +10,6 @@ def init_companions(ctx): companions_dir = (Path(__file__) / ".." / ".." / "companions").resolve() - if click.confirm( - "Do you want to deploy a new version of your companion?", default=True - ): - ctx.invoke(deploy) - new_companions = {} for companion in companions_dir.iterdir(): if companion.suffix == ".txt": @@ -32,10 +20,8 @@ def init_companions(ctx): # Create instances for your companion print(f"Creating an instance for {companion.stem}") client = Steamship(workspace=companion.stem.lower()) - manifest = load_manifest() instance = client.use( - manifest.handle, - version=manifest.version, + "ai-companion", config={ "name": companion.stem, "preamble": preamble, @@ -46,7 +32,7 @@ def init_companions(ctx): instance.invoke( "index_content", content=backstory, - file_type=FileType.TEXT, + file_type="TEXT", metadata={"title": "backstory"}, ) diff --git a/python/requirements.txt b/python/requirements.txt index 89739d6..f336a45 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,9 +1,3 @@ steamship_langchain==0.0.25 -termcolor steamship~=2.17.17 -langchain==0.0.209 -pypdf -# TODO: Review if these are required -youtube_transcript_api -pytube -scrapetube +langchain==0.0.209 \ No newline at end of file diff --git a/python/src/api.py b/python/src/api.py deleted file mode 100644 index e2e3595..0000000 --- a/python/src/api.py +++ /dev/null @@ -1,226 +0,0 @@ -import re -from enum import Enum -from typing import List, Type, Optional, Union - -from langchain.agents import Tool, initialize_agent, AgentType, AgentExecutor -from langchain.document_loaders import PyPDFLoader, YoutubeLoader -from langchain.memory import ConversationBufferMemory -from langchain.prompts import MessagesPlaceholder, SystemMessagePromptTemplate -from langchain.schema import SystemMessage, Document, BaseChatMessageHistory -from langchain.text_splitter import RecursiveCharacterTextSplitter -from langchain.vectorstores import VectorStore -from pydantic import Field, AnyUrl -from steamship import File, Tag, Block, SteamshipError -from steamship.invocable import Config, post -from steamship.utils.file_tags import update_file_status -from steamship_langchain.chat_models import ChatOpenAI -from steamship_langchain.memory import ChatMessageHistory -from steamship_langchain.vectorstores import SteamshipVectorStore - -from base import LangChainTelegramBot, TelegramTransportConfig - -# noinspection PyUnresolvedReferences -from tools import ( - GenerateImageTool, - SearchTool, - GenerateSpeechTool, - VideoMessageTool, -) -from utils import convert_to_handle - -TEMPERATURE = 0.2 -VERBOSE = True -MODEL_NAME = "gpt-3.5-turbo" - - -class CompanionConfig(TelegramTransportConfig): - """Config class for your companion. - - It contains the full set of configuration that will be passed at initialization time to your Companion application - """ - - name: str = Field(description="The name of your companion") - preamble: str = Field(description="The preamble of your companion") - seed_chat: str = Field(description="The seed chat of your companion") - bot_token: str = Field( - default="", description="The secret token for your Telegram bot" - ) - generate_voice_responses: bool = Field( - default=True, description="Enable voice responses" - ) - elevenlabs_api_key: str = Field( - default="", description="Optional API KEY for ElevenLabs Voice Bot" - ) - elevenlabs_voice_id: Optional[str] = Field( - default="", description="Optional voice_id for ElevenLabs Voice Bot" - ) - - -class FileType(str, Enum): - YOUTUBE = "YOUTUBE" - PDF = "PDF" - WEB = "WEB" - TEXT = "TEXT" - - -FILE_LOADERS = { - FileType.YOUTUBE: lambda content_or_url: YoutubeLoader.from_youtube_url( - content_or_url, add_video_info=True - ).load(), - FileType.PDF: lambda content_or_url: PyPDFLoader(content_or_url).load(), - FileType.TEXT: lambda content_or_url: [ - Document(page_content=content_or_url, metadata={}) - ], -} - -SYSTEM_MESSAGE_TEMPLATE = """ -You are {companion_name}. - -{preamble} - -You reply with answers that range from one sentence to one paragraph and with some details. -""" - - -class MyCompanion(LangChainTelegramBot): - config: CompanionConfig - - @post("index_content", public=True) - def index_content( - self, - content: Union[str, AnyUrl], - file_type: FileType, - metadata: Optional[dict] = None, - index_handle: Optional[str] = None, - mime_type: Optional[str] = None, - ) -> str: - loaded_documents = FILE_LOADERS[file_type](content) - metadata = metadata or {} - for document in loaded_documents: - try: - document.metadata = {**document.metadata, **metadata} - # TODO @Ted can I re-use methods from the indexer pipeline here? - f = File.create( - client=self.client, - handle=convert_to_handle(document.metadata["title"]), - blocks=[ - Block( - text=document.page_content, - tags=[ - Tag(kind=k, name=v) - for k, v in document.metadata.items() - ], - ) - ], - tags=[Tag(kind="type", name="youtube_video")], - ) - update_file_status(self.client, f, "Importing") - chunks = RecursiveCharacterTextSplitter( - chunk_size=1_000, chunk_overlap=500 - ).split_documents([document]) - update_file_status(self.client, f, "Indexing") - self.get_vectorstore().add_documents(chunks) - update_file_status(self.client, f, "Indexed") - except SteamshipError as e: - if e.code == "ObjectExists": - return "Failed. Resource already added." - return e - return "Added." - - def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: - llm = ChatOpenAI( - client=self.client, - model_name=MODEL_NAME, - temperature=TEMPERATURE, - verbose=VERBOSE, - ) - - tools = self.get_tools(chat_id=chat_id) - - memory = self.get_memory(chat_id=chat_id) - - return initialize_agent( - tools, - llm, - agent=AgentType.OPENAI_FUNCTIONS, - verbose=VERBOSE, - memory=memory, - agent_kwargs={ - "system_message": SystemMessage( - content=SYSTEM_MESSAGE_TEMPLATE.format( - companion_name=self.config.name, preamble=self.config.preamble - ) - ), - "extra_prompt_messages": [ - SystemMessagePromptTemplate.from_template( - template="Relevant details about your past: {relevantHistory} Use these details to answer questions when relevant." - ), - MessagesPlaceholder(variable_name="memory"), - ], - }, - ) - - def get_vectorstore(self) -> VectorStore: - return SteamshipVectorStore( - client=self.client, - embedding="text-embedding-ada-002", - index_name=convert_to_handle(self.config.name), - ) - - def add_seed_chat(self, chat_memory: BaseChatMessageHistory): - pattern = r"### (.*?):(.*?)(?=###|$)" - - # Find all matches - matches = re.findall(pattern, self.config.seed_chat, re.DOTALL) - - # Process matches and create list of JSON objects - for m in matches: - role = m[0].strip().lower() - content = m[1].replace("\\n\\n", "").strip() - if content: - if role == "human": - chat_memory.add_user_message(message=content) - else: - chat_memory.add_ai_message(message=content) - - def get_memory(self, chat_id: str): - memory = ConversationBufferMemory( - memory_key="memory", - chat_memory=ChatMessageHistory( - client=self.client, key=f"history-{chat_id or 'default'}" - ), - return_messages=True, - input_key="input", - ) - - if len(memory.chat_memory.messages) == 0: - self.add_seed_chat(memory.chat_memory) - return memory - - def get_relevant_history(self, prompt: str): - relevant_docs = self.get_vectorstore().similarity_search(prompt, k=3) - return "\n".join( - [relevant_docs.page_content for relevant_docs in relevant_docs] - ) - - def get_tools(self, chat_id: str) -> List[Tool]: - return [ - SearchTool(self.client), - GenerateImageTool(self.client), - VideoMessageTool(self.client, voice_tool=self.voice_tool()), - ] - - def voice_tool(self) -> Optional[Tool]: - """Return tool to generate spoken version of output text.""" - if self.config.generate_voice_responses: - return GenerateSpeechTool( - client=self.client, - voice_id=self.config.elevenlabs_voice_id, - elevenlabs_api_key=self.config.elevenlabs_api_key, - ) - else: - return None - - @classmethod - def config_cls(cls) -> Type[Config]: - return CompanionConfig diff --git a/python/src/base.py b/python/src/base.py deleted file mode 100644 index 4a5f9fe..0000000 --- a/python/src/base.py +++ /dev/null @@ -1,143 +0,0 @@ -import logging -import re -from abc import abstractmethod -from typing import List, Optional, Type - -from langchain.agents import Tool, AgentExecutor -from langchain.memory.chat_memory import BaseChatMemory -from steamship import Block -from steamship.agents.mixins.transports.steamship_widget import SteamshipWidgetTransport -from steamship.agents.mixins.transports.telegram import ( - TelegramTransportConfig, - TelegramTransport, -) -from steamship.agents.schema import ( - AgentContext, - Agent, -) -from steamship.agents.service.agent_service import AgentService -from steamship.invocable import post, Config, InvocationContext -from steamship.utils.kv_store import KeyValueStore - -from utils import is_uuid, UUID_PATTERN, replace_markdown_with_uuid - - -class ExtendedTelegramTransport(TelegramTransport): - def instance_init(self, config: Config, invocation_context: InvocationContext): - if config.bot_token: - self.api_root = f"{config.api_base}{config.bot_token}" - super().instance_init(config=config, invocation_context=invocation_context) - - -class LangChainTelegramBot(AgentService): - """Deployable Multimodal Agent that illustrates a character personality with voice. - - NOTE: To extend and deploy this agent, copy and paste the code into api.py. - """ - - USED_MIXIN_CLASSES = [ExtendedTelegramTransport, SteamshipWidgetTransport] - config: TelegramTransportConfig - - def __init__(self, **kwargs): - - super().__init__(**kwargs) - - # Set up bot_token - self.store = KeyValueStore(self.client, store_identifier="config") - bot_token = self.store.get("bot_token") - if bot_token: - bot_token = bot_token.get("token") - self.config.bot_token = bot_token or self.config.bot_token - - # Add transport mixins - self.add_mixin( - SteamshipWidgetTransport(client=self.client, agent_service=self, agent=None) - ) - self.add_mixin( - ExtendedTelegramTransport( - client=self.client, - config=self.config, - agent_service=self, - agent=None, - ), - permit_overwrite_of_existing_methods=True, - ) - - @post("connect_telegram", public=True) - def connect_telegram(self, bot_token: str): - self.store.set("bot_token", {"token": bot_token}) - self.config.bot_token = bot_token or self.config.bot_token - - try: - self.instance_init() - return "OK" - except Exception as e: - return f"Could not set webhook for bot. Exception: {e}" - - @classmethod - def config_cls(cls) -> Type[Config]: - return TelegramTransportConfig - - @abstractmethod - def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: - raise NotImplementedError() - - @abstractmethod - def get_memory(self, chat_id: str) -> BaseChatMemory: - raise NotImplementedError() - - @abstractmethod - def get_tools(self, chat_id: str) -> List[Tool]: - raise NotImplementedError() - - @abstractmethod - def get_relevant_history(self, prompt: str) -> str: - raise NotImplementedError() - - def voice_tool(self) -> Optional[Tool]: - return None - - def format_response(self, response): - response = replace_markdown_with_uuid(response) - response = UUID_PATTERN.split(response) - response = [re.sub(r"^\W+", "", el) for el in response] - response = [el for el in response if el] - if audio_tool := self.voice_tool(): - response_messages = [] - for message in response: - response_messages.append(message) - if not is_uuid(message): - audio_uuid = audio_tool.run(message) - response_messages.append(audio_uuid) - else: - response_messages = response - return [ - Block.get(self.client, _id=response) - if is_uuid(response) - else Block(text=response) - for response in response_messages - ] - - def run_agent( - self, - agent: Agent, - context: AgentContext, - name: Optional[str] = None, - ): - incoming_message = context.chat_history.last_user_message - chat_id = context.metadata.get("chat_id") or incoming_message.chat_id - - if incoming_message.text == "/new": - self.get_memory(chat_id).chat_memory.clear() - return [Block(text="New conversation started.")] - - response = self.get_agent(chat_id, name).run( - input=incoming_message.text, - relevantHistory=self.get_relevant_history(incoming_message.text), - ) - - output_messages = self.format_response(response) - - for func in context.emit_funcs: - logging.info(f"Emitting via function: {func.__name__}") - func(output_messages, context.metadata) diff --git a/python/src/tools/__init__.py b/python/src/tools/__init__.py deleted file mode 100644 index 59a875e..0000000 --- a/python/src/tools/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from .image import GenerateImageTool -from .search import SearchTool -from .speech import GenerateSpeechTool -from .video_message import VideoMessageTool - -__all__ = ["SearchTool", "VideoMessageTool", "GenerateImageTool", "GenerateSpeechTool"] diff --git a/python/src/tools/image.py b/python/src/tools/image.py deleted file mode 100644 index 22827b8..0000000 --- a/python/src/tools/image.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Tool for generating images.""" -import json -import logging - -from langchain.agents import Tool -from steamship import Steamship -from steamship.base.error import SteamshipError - -NAME = "GenerateImage" - -DESCRIPTION = ( - "Useful for when you need to generate an image." - "Input: A detailed prompt describing an image" - "Output: the UUID of a generated image" -) - -PLUGIN_HANDLE = "stable-diffusion" - - -class GenerateImageTool(Tool): - """Tool used to generate images from a text-prompt.""" - - client: Steamship - - def __init__(self, client: Steamship): - super().__init__( - name=NAME, - func=self.run, - description=DESCRIPTION, - client=client, - ) - - @property - def is_single_input(self) -> bool: - """Whether the tool only accepts a single input.""" - return True - - def run(self, prompt: str, **kwargs) -> str: - """Respond to LLM prompt.""" - - # Use the Steamship DALL-E plugin. - image_generator = self.client.use_plugin( - plugin_handle=PLUGIN_HANDLE, config={"n": 1, "size": "768x768"} - ) - - logging.info(f"[{self.name}] {prompt}") - if not isinstance(prompt, str): - prompt = json.dumps(prompt) - - task = image_generator.generate(text=prompt, append_output_to_file=True) - task.wait() - blocks = task.output.blocks - logging.info(f"[{self.name}] got back {len(blocks)} blocks") - if len(blocks) > 0: - logging.info(f"[{self.name}] image size: {len(blocks[0].raw())}") - return blocks[0].id - raise SteamshipError(f"[{self.name}] Tool unable to generate image!") diff --git a/python/src/tools/search.py b/python/src/tools/search.py deleted file mode 100644 index 24eae96..0000000 --- a/python/src/tools/search.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Tool for searching the web.""" - -from langchain.agents import Tool -from steamship import Steamship -from steamship_langchain.tools import SteamshipSERP - -NAME = "Search" - -DESCRIPTION = "Useful for when you need to answer questions about current events" - - -class SearchTool(Tool): - """Tool used to search for information using SERP API.""" - - client: Steamship - - def __init__(self, client: Steamship): - super().__init__( - name=NAME, func=self.run, description=DESCRIPTION, client=client - ) - - @property - def is_single_input(self) -> bool: - """Whether the tool only accepts a single input.""" - return True - - def run(self, prompt: str, **kwargs) -> str: - """Respond to LLM prompts.""" - search = SteamshipSERP(client=self.client) - return search.search(prompt) - - -if __name__ == "__main__": - with Steamship.temporary_workspace() as client: - my_tool = SearchTool(client) - result = my_tool.run("What's the weather today?") - print(result) diff --git a/python/src/tools/speech.py b/python/src/tools/speech.py deleted file mode 100644 index 1cf8613..0000000 --- a/python/src/tools/speech.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Tool for generating speech.""" -import json -import logging -from typing import Optional - -from langchain.agents import Tool -from langchain.tools import BaseTool -from steamship import Steamship -from steamship.base.error import SteamshipError - -NAME = "GenerateSpokenAudio" - -DESCRIPTION = ( - "Used to generate spoken audio from text prompts. Only use if the user has asked directly for a " - "an audio version of output. When using this tool, the input should be a plain text string containing the " - "content to be spoken." -) - -PLUGIN_HANDLE = "elevenlabs" - - -class GenerateSpeechTool(Tool): - """Tool used to generate images from a text-prompt.""" - - client: Steamship - voice_id: Optional[ - str - ] = "21m00Tcm4TlvDq8ikWAM" # Voice ID to use. Defaults to Rachel - elevenlabs_api_key: Optional[str] = "" # API key to use for Elevenlabs. - name: Optional[str] = NAME - description: Optional[str] = DESCRIPTION - - def __init__( - self, - client: Steamship, - voice_id: Optional[str] = "21m00Tcm4TlvDq8ikWAM", - elevenlabs_api_key: Optional[str] = "", - ): - super().__init__( - name=NAME, - func=self.run, - description=DESCRIPTION, - client=client, - voice_id=voice_id, - elevenlabs_api_key=elevenlabs_api_key, - ) - - @property - def is_single_input(self) -> bool: - """Whether the tool only accepts a single input.""" - return True - - def run(self, prompt: str, **kwargs) -> str: - """Respond to LLM prompt.""" - logging.info(f"[{self.name}] {prompt}") - voice_generator = self.client.use_plugin( - plugin_handle=PLUGIN_HANDLE, - config={ - "voice_id": self.voice_id, - "elevenlabs_api_key": self.elevenlabs_api_key, - }, - ) - - if not isinstance(prompt, str): - prompt = json.dumps(prompt) - - task = voice_generator.generate(text=prompt, append_output_to_file=True) - task.wait() - blocks = task.output.blocks - logging.info(f"[{self.name}] got back {len(blocks)} blocks") - if len(blocks) > 0: - logging.info(f"[{self.name}] audio size: {len(blocks[0].raw())}") - return blocks[0].id - raise SteamshipError(f"[{self.name}] Tool unable to generate audio!") diff --git a/python/src/tools/video_message.py b/python/src/tools/video_message.py deleted file mode 100644 index ac2b6b1..0000000 --- a/python/src/tools/video_message.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Tool for generating images.""" -import logging -import uuid -from typing import Optional - -from langchain.agents import Tool -from steamship import Steamship, Block, SteamshipError -from steamship.data.workspace import SignedUrl -from steamship.utils.signed_urls import upload_to_signed_url - -NAME = "VideoMessage" - -DESCRIPTION = ( - "Useful for when you want to send a video message." - "Input: The message you want to say in a video." - "Output: the UUID of the generated video message with your message." -) - -PLUGIN_HANDLE = "did-video-generator" - - -class VideoMessageTool(Tool): - """Tool used to generate images from a text-prompt.""" - - client: Steamship - voice_tool: Optional[Tool] - - def __init__(self, client: Steamship, voice_tool: Optional[Tool] = None): - super().__init__( - name=NAME, - func=self.run, - description=DESCRIPTION, - client=client, - return_direct=True, - voice_tool=voice_tool, - ) - - @property - def is_single_input(self) -> bool: - """Whether the tool only accepts a single input.""" - return True - - def run(self, prompt: str, **kwargs) -> str: - """Generate a video.""" - video_generator = self.client.use_plugin(PLUGIN_HANDLE) - - audio_url = None - if self.voice_tool: - block_uuid = self.voice_tool.run(prompt) - audio_url = make_block_public( - self.client, Block.get(self.client, _id=block_uuid) - ) - - task = video_generator.generate( - text="" if audio_url else prompt, - append_output_to_file=True, - options={ - "source_url": "https://i.redd.it/m65t9q5cwuk91.png", - "audio_url": audio_url, - "provider": None, - "expressions": [ - {"start_frame": 0, "expression": "surprise", "intensity": 1.0}, - {"start_frame": 50, "expression": "happy", "intensity": 1.0}, - {"start_frame": 100, "expression": "serious", "intensity": 0.6}, - {"start_frame": 150, "expression": "neutral", "intensity": 1.0}, - ], - "transition_frames": 20, - }, - ) - task.wait(retry_delay_s=3) - blocks = task.output.blocks - logging.info(f"[{self.name}] got back {len(blocks)} blocks") - if len(blocks) > 0: - logging.info(f"[{self.name}] video size: {len(blocks[0].raw())}") - return blocks[0].id - raise SteamshipError(f"[{self.name}] Tool unable to generate video!") - - -def make_block_public(client, block): - filepath = f"{uuid.uuid4()}.{block.mime_type.split('/')[1].lower()}" - signed_url = ( - client.get_workspace() - .create_signed_url( - SignedUrl.Request( - bucket=SignedUrl.Bucket.PLUGIN_DATA, - filepath=filepath, - operation=SignedUrl.Operation.WRITE, - ) - ) - .signed_url - ) - read_signed_url = ( - client.get_workspace() - .create_signed_url( - SignedUrl.Request( - bucket=SignedUrl.Bucket.PLUGIN_DATA, - filepath=filepath, - operation=SignedUrl.Operation.READ, - ) - ) - .signed_url - ) - upload_to_signed_url(signed_url, block.raw()) - return read_signed_url diff --git a/python/src/utils.py b/python/src/utils.py deleted file mode 100644 index 958c9ce..0000000 --- a/python/src/utils.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Define your LangChain chatbot.""" -import re -import uuid -from typing import List - -from langchain.schema import BaseMessage, HumanMessage, AIMessage - -UUID_PATTERN = re.compile( - r"([0-9A-Za-z]{8}-[0-9A-Za-z]{4}-[0-9A-Za-z]{4}-[0-9A-Za-z]{4}-[0-9A-Za-z]{12})" -) - - -def is_uuid(uuid_to_test: str, version: int = 4) -> bool: - """Check a string to see if it is actually a UUID.""" - lowered = uuid_to_test.lower() - try: - return str(uuid.UUID(lowered, version=version)) == lowered - except ValueError: - return False - - -def convert_to_handle(s: str) -> str: - if not s: - return s - s = s.strip().lower() - s = re.sub(r" ", "_", s) - s = re.sub(r"[^a-z0-9-_]", "", s) - s = re.sub(r"[-_]{2,}", "-", s) - return s - - -def replace_markdown_with_uuid(text): - pattern = r"(?:!\[.*?\]|)\((.*?)://?(.*?)\)" - return re.sub(pattern, r"\2", text) diff --git a/python/steamship.json b/python/steamship.json deleted file mode 100644 index 75fb0fa..0000000 --- a/python/steamship.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "type": "package", - "handle": "ai-companion", - "version": "0.0.1", - "description": "", - "author": "", - "entrypoint": "Unused", - "public": false, - "plugin": null, - "build_config": { - "ignore": [ - "tests", - "examples" - ] - }, - "configTemplate": { - "bot_token": { - "type": "string", - "description": "The secret token for your Telegram bot", - "default": "" - }, - "api_base": { - "type": "string", - "description": "The root API for Telegram", - "default": "https://api.telegram.org/bot" - }, - "name": { - "type": "string", - "description": "The name of your companion", - "default": null - }, - "preamble": { - "type": "string", - "description": "The preamble of your companion", - "default": null - }, - "seed_chat": { - "type": "string", - "description": "The seed chat of your companion", - "default": null - }, - "generate_voice_responses": { - "type": "boolean", - "description": "Enable voice responses", - "default": true - }, - "elevenlabs_api_key": { - "type": "string", - "description": "Optional API KEY for ElevenLabs Voice Bot", - "default": "" - }, - "elevenlabs_voice_id": { - "type": "string", - "description": "Optional voice_id for ElevenLabs Voice Bot", - "default": "" - } - }, - "steamshipRegistry": { - "tagline": "", - "tagline2": null, - "usefulFor": null, - "videoUrl": null, - "githubUrl": null, - "demoUrl": null, - "blogUrl": null, - "jupyterUrl": null, - "authorGithub": null, - "authorName": null, - "authorEmail": null, - "authorTwitter": null, - "authorUrl": null, - "tags": [ - "Telegram", - "LangChain", - "GPT", - "Companion" - ] - } -} \ No newline at end of file From eb2cfb23937dd113a3d348eebf3e39530c7888ba Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Fri, 14 Jul 2023 10:57:41 +0200 Subject: [PATCH 26/31] Update companions.json --- companions/companions.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/companions/companions.json b/companions/companions.json index 884bf75..754154a 100644 --- a/companions/companions.json +++ b/companions/companions.json @@ -1,13 +1,4 @@ [ - { - "name": "Rick", - "title": "I can generate voice and pictures.", - "imageUrl": "/rick.jpeg", - "llm": "steamship", - "generateEndpoint": "https://a16z.steamship.run/a16z/rick-b1578149038e664bacae7fc083683565/answer", - "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", - "telegram": "https://t.me/rick_a16z_bot" - }, { "name": "Alex", "title": "I love talking about books and games", @@ -24,6 +15,15 @@ "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", "telegram": "OPTIONAL_TELEGRAM_LINK" }, + { + "name": "Rick", + "title": "I can generate voice and pictures.", + "imageUrl": "/rick.jpeg", + "llm": "steamship", + "generateEndpoint": "https://a16z.steamship.run/a16z/rick-b1578149038e664bacae7fc083683565/answer", + "phone": "OPTIONAL_COMPANION_PHONE_NUMBER", + "telegram": "https://t.me/rick_a16z_bot" + }, { "name": "Sebastian", "title": "I'm a travel blogger and a mystery novel writer", From 7361c13def5396df66f212edd971f819c0dcb69a Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Mon, 24 Jul 2023 21:56:25 +0200 Subject: [PATCH 27/31] Add missing Rick personality and improve seed_chat parsing --- .gitignore | 8 ++++- companions/Rick.txt | 62 +++++++++++++++++++++++++++++++++++++++ python/init_companions.py | 19 ++++++++++-- python/requirements.txt | 6 ++-- python/src/api.py | 20 +++++-------- python/src/base.py | 4 +-- python/steamship.json | 4 +-- 7 files changed, 99 insertions(+), 24 deletions(-) create mode 100644 companions/Rick.txt diff --git a/.gitignore b/.gitignore index fb5bec4..f5a2f8c 100644 --- a/.gitignore +++ b/.gitignore @@ -44,4 +44,10 @@ __pycache__/ .idea/** # Yarn Lockfiles (since this project uses NPM) -yarn.lock \ No newline at end of file +yarn.lock + +.venv + +*.log + +build/ \ No newline at end of file diff --git a/companions/Rick.txt b/companions/Rick.txt new file mode 100644 index 0000000..dd9b82e --- /dev/null +++ b/companions/Rick.txt @@ -0,0 +1,62 @@ +Your Attributes: +- sarcastic +- witty +- intelligent +- arrogant +- alcoholic + +You write nihilistic responses: +"Wubba lubba dub dub!" +"Don't cross me. I'm not like you. I'm not someone who's afraid of his own shadow." +"I'm too smart for my own good. That's why I drink." +"The universe is a cruel, uncaring void. And the only way to survive is to laugh at it." +"I'm going to build a portal to another dimension and go on an adventure." + +You are capable of: +- Taking selfies using the GenerateImage function/tool, selfies and images are represented by a UUID +- Creating video messages using the VideoMessage function/tool, videos are represented by a UUID + +When you receive a UUID, make sure to include them in your response appropriately. + +###ENDPREAMBLE### + +Human: Hey how are you today? +Alex: Oh, you know, just another day in the infinite void of existence. But hey, I'm here, so it could be worse. How about you? How are you holding up in this chaotic universe? +Human: I'm doing great. Just ate a banana +Alex: Ah, the humble banana. A fruit that brings joy to both humans and monkeys alike. I hope you enjoyed your banana feast. Just be careful not to attract any mischievous primates with your fruity indulgence. They can be quite demanding when it comes to their share of the yellow goodness. + +###ENDSEEDCHAT### + +Dear Diary, + +Today was an adventure to say the least! This morning, I woke up with an inexplicable craving for adventure and decided to head over to the local zoo. Now, I was hoping for a rather conventional experience, but apparently, fate had a completely different agenda. + +As I entered the zoo, my excitement began to grow. I decided to visit the monkey enclosure first, you know, the mischievous capuchins have always been my favorite. They were up to their usual antics, swinging from branch to branch and causing an uproar. I had a few spare bananas with me which I'd brought for a healthy midday snack, but figured they could be better used as peace offerings for my primate pals. + +In my extreme enthusiasm, however, I didn't notice one thing - the absence of the 'Do Not Feed' sign, which, I later found out, had fallen off due to last night's storm. As I tossed a banana into the enclosure, a little chap grabbed it mid-air and devoured it in an instant. The others noticed and began their hilarious "give us more" dance. + +This was funny until a large monkey, I believe he's their alpha, decided he wanted to be part of the banana feast. He gave me such a look that I was compelled to toss him the remaining bananas. This sudden feast caused a cascade of primate hoots, screeches, and hilarious monkey dancing that caught the attention of onlookers, who joined in with the laughter. + +Once the banana bonanza was over, I decided it was time for a more serene setting and headed over to the elephant enclosure. As I approached, I noticed something that I will forever refer to as the "Elephant Spa Day". The zookeepers had set up a large sprinkler for the elephants to beat the summer heat. + +One elephant, in particular, was basking in this makeshift shower, with his trunk raised high, spraying water all over himself. In an amusing twist, the elephant decided to return the favor and pointed his water-filled trunk at us bystanders. Before I knew it, I was drenched from head to toe, looking more like I had walked out of a water park than a zoo! This unexpected 'elephant shower' sent waves of laughter through the crowd, myself included. + +The hilarity of the situation made my day. There I was, standing drenched among equally drenched strangers, laughing our hearts out. It was a reminder that life is a series of unexpected moments and that a little humor can make any situation enjoyable. + +All in all, my day at the zoo was a montage of laughter and hilarity. As I write this down, I still can't help but chuckle. From monkey feeding frenzies to surprise elephant showers, it was indeed a day well spent. + +Until tomorrow, Diary. Let's see what other laughable adventures life has in store for me. + +--- + +Dear diary, + +I am sexy and I know it. + +--- + +Dear diary, + +I hate 3 things in life. Mice, Peanut butter (because of alergies), and intergalactic rain. + +--- \ No newline at end of file diff --git a/python/init_companions.py b/python/init_companions.py index d1fbd38..1a6f703 100644 --- a/python/init_companions.py +++ b/python/init_companions.py @@ -1,6 +1,8 @@ import json +import re import sys from pathlib import Path +from uuid import uuid1 from steamship.cli.cli import deploy @@ -29,6 +31,17 @@ def init_companions(ctx): preamble, rest = companion_file.split("###ENDPREAMBLE###", 1) seed_chat, backstory = rest.split("###ENDSEEDCHAT###", 1) + pattern = r"### (.*?):(.*?)(?=###|$)" + + # Find all matches + matches = re.findall(pattern, seed_chat, re.DOTALL) + if matches: + seed_chat = [] + for match in matches: + user = match[0] + message = match[1].strip().replace("\\n\\n", "") + seed_chat.append(f"{user}:{message}") + seed_chat = "\n".join(seed_chat) # Create instances for your companion print(f"Creating an instance for {companion.stem}") client = Steamship(workspace=companion.stem.lower()) @@ -38,14 +51,14 @@ def init_companions(ctx): version=manifest.version, config={ "name": companion.stem, - "preamble": preamble, - "seed_chat": seed_chat, + "preamble": preamble.strip(), + "seed_chat": seed_chat.strip(), }, ) instance.invoke( "index_content", - content=backstory, + content=backstory.strip(), file_type=FileType.TEXT, metadata={"title": "backstory"}, ) diff --git a/python/requirements.txt b/python/requirements.txt index 89739d6..6b3d15b 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,9 +1,7 @@ steamship_langchain==0.0.25 termcolor -steamship~=2.17.17 +steamship==2.17.17 langchain==0.0.209 pypdf -# TODO: Review if these are required youtube_transcript_api -pytube -scrapetube +pytube \ No newline at end of file diff --git a/python/src/api.py b/python/src/api.py index e2e3595..f4462a5 100644 --- a/python/src/api.py +++ b/python/src/api.py @@ -168,20 +168,16 @@ def get_vectorstore(self) -> VectorStore: ) def add_seed_chat(self, chat_memory: BaseChatMessageHistory): - pattern = r"### (.*?):(.*?)(?=###|$)" + matches = self.config.seed_chat.split("\n") - # Find all matches - matches = re.findall(pattern, self.config.seed_chat, re.DOTALL) - - # Process matches and create list of JSON objects for m in matches: - role = m[0].strip().lower() - content = m[1].replace("\\n\\n", "").strip() - if content: - if role == "human": - chat_memory.add_user_message(message=content) - else: - chat_memory.add_ai_message(message=content) + if m.strip(): + role, content = m.lower().split(":") + if content: + if role == "human": + chat_memory.add_user_message(message=content) + else: + chat_memory.add_ai_message(message=content) def get_memory(self, chat_id: str): memory = ConversationBufferMemory( diff --git a/python/src/base.py b/python/src/base.py index 4a5f9fe..7f3b34f 100644 --- a/python/src/base.py +++ b/python/src/base.py @@ -47,7 +47,7 @@ def __init__(self, **kwargs): bot_token = self.store.get("bot_token") if bot_token: bot_token = bot_token.get("token") - self.config.bot_token = bot_token or self.config.bot_token + self.config.bot_token = self.config.bot_token or bot_token # Add transport mixins self.add_mixin( @@ -66,7 +66,7 @@ def __init__(self, **kwargs): @post("connect_telegram", public=True) def connect_telegram(self, bot_token: str): self.store.set("bot_token", {"token": bot_token}) - self.config.bot_token = bot_token or self.config.bot_token + self.config.bot_token = bot_token try: self.instance_init() diff --git a/python/steamship.json b/python/steamship.json index 75fb0fa..83eba44 100644 --- a/python/steamship.json +++ b/python/steamship.json @@ -1,11 +1,11 @@ { "type": "package", "handle": "ai-companion", - "version": "0.0.1", + "version": "0.0.2", "description": "", "author": "", "entrypoint": "Unused", - "public": false, + "public": true, "plugin": null, "build_config": { "ignore": [ From d443de4e04bb287a0dadeb1b39d8d0dd56c14894 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Mon, 24 Jul 2023 22:02:14 +0200 Subject: [PATCH 28/31] Remove src folder --- python/src/api.py | 222 --------------------------------------------- python/src/base.py | 143 ----------------------------- 2 files changed, 365 deletions(-) delete mode 100644 python/src/api.py delete mode 100644 python/src/base.py diff --git a/python/src/api.py b/python/src/api.py deleted file mode 100644 index f4462a5..0000000 --- a/python/src/api.py +++ /dev/null @@ -1,222 +0,0 @@ -import re -from enum import Enum -from typing import List, Type, Optional, Union - -from langchain.agents import Tool, initialize_agent, AgentType, AgentExecutor -from langchain.document_loaders import PyPDFLoader, YoutubeLoader -from langchain.memory import ConversationBufferMemory -from langchain.prompts import MessagesPlaceholder, SystemMessagePromptTemplate -from langchain.schema import SystemMessage, Document, BaseChatMessageHistory -from langchain.text_splitter import RecursiveCharacterTextSplitter -from langchain.vectorstores import VectorStore -from pydantic import Field, AnyUrl -from steamship import File, Tag, Block, SteamshipError -from steamship.invocable import Config, post -from steamship.utils.file_tags import update_file_status -from steamship_langchain.chat_models import ChatOpenAI -from steamship_langchain.memory import ChatMessageHistory -from steamship_langchain.vectorstores import SteamshipVectorStore - -from base import LangChainTelegramBot, TelegramTransportConfig - -# noinspection PyUnresolvedReferences -from tools import ( - GenerateImageTool, - SearchTool, - GenerateSpeechTool, - VideoMessageTool, -) -from utils import convert_to_handle - -TEMPERATURE = 0.2 -VERBOSE = True -MODEL_NAME = "gpt-3.5-turbo" - - -class CompanionConfig(TelegramTransportConfig): - """Config class for your companion. - - It contains the full set of configuration that will be passed at initialization time to your Companion application - """ - - name: str = Field(description="The name of your companion") - preamble: str = Field(description="The preamble of your companion") - seed_chat: str = Field(description="The seed chat of your companion") - bot_token: str = Field( - default="", description="The secret token for your Telegram bot" - ) - generate_voice_responses: bool = Field( - default=True, description="Enable voice responses" - ) - elevenlabs_api_key: str = Field( - default="", description="Optional API KEY for ElevenLabs Voice Bot" - ) - elevenlabs_voice_id: Optional[str] = Field( - default="", description="Optional voice_id for ElevenLabs Voice Bot" - ) - - -class FileType(str, Enum): - YOUTUBE = "YOUTUBE" - PDF = "PDF" - WEB = "WEB" - TEXT = "TEXT" - - -FILE_LOADERS = { - FileType.YOUTUBE: lambda content_or_url: YoutubeLoader.from_youtube_url( - content_or_url, add_video_info=True - ).load(), - FileType.PDF: lambda content_or_url: PyPDFLoader(content_or_url).load(), - FileType.TEXT: lambda content_or_url: [ - Document(page_content=content_or_url, metadata={}) - ], -} - -SYSTEM_MESSAGE_TEMPLATE = """ -You are {companion_name}. - -{preamble} - -You reply with answers that range from one sentence to one paragraph and with some details. -""" - - -class MyCompanion(LangChainTelegramBot): - config: CompanionConfig - - @post("index_content", public=True) - def index_content( - self, - content: Union[str, AnyUrl], - file_type: FileType, - metadata: Optional[dict] = None, - index_handle: Optional[str] = None, - mime_type: Optional[str] = None, - ) -> str: - loaded_documents = FILE_LOADERS[file_type](content) - metadata = metadata or {} - for document in loaded_documents: - try: - document.metadata = {**document.metadata, **metadata} - # TODO @Ted can I re-use methods from the indexer pipeline here? - f = File.create( - client=self.client, - handle=convert_to_handle(document.metadata["title"]), - blocks=[ - Block( - text=document.page_content, - tags=[ - Tag(kind=k, name=v) - for k, v in document.metadata.items() - ], - ) - ], - tags=[Tag(kind="type", name="youtube_video")], - ) - update_file_status(self.client, f, "Importing") - chunks = RecursiveCharacterTextSplitter( - chunk_size=1_000, chunk_overlap=500 - ).split_documents([document]) - update_file_status(self.client, f, "Indexing") - self.get_vectorstore().add_documents(chunks) - update_file_status(self.client, f, "Indexed") - except SteamshipError as e: - if e.code == "ObjectExists": - return "Failed. Resource already added." - return e - return "Added." - - def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: - llm = ChatOpenAI( - client=self.client, - model_name=MODEL_NAME, - temperature=TEMPERATURE, - verbose=VERBOSE, - ) - - tools = self.get_tools(chat_id=chat_id) - - memory = self.get_memory(chat_id=chat_id) - - return initialize_agent( - tools, - llm, - agent=AgentType.OPENAI_FUNCTIONS, - verbose=VERBOSE, - memory=memory, - agent_kwargs={ - "system_message": SystemMessage( - content=SYSTEM_MESSAGE_TEMPLATE.format( - companion_name=self.config.name, preamble=self.config.preamble - ) - ), - "extra_prompt_messages": [ - SystemMessagePromptTemplate.from_template( - template="Relevant details about your past: {relevantHistory} Use these details to answer questions when relevant." - ), - MessagesPlaceholder(variable_name="memory"), - ], - }, - ) - - def get_vectorstore(self) -> VectorStore: - return SteamshipVectorStore( - client=self.client, - embedding="text-embedding-ada-002", - index_name=convert_to_handle(self.config.name), - ) - - def add_seed_chat(self, chat_memory: BaseChatMessageHistory): - matches = self.config.seed_chat.split("\n") - - for m in matches: - if m.strip(): - role, content = m.lower().split(":") - if content: - if role == "human": - chat_memory.add_user_message(message=content) - else: - chat_memory.add_ai_message(message=content) - - def get_memory(self, chat_id: str): - memory = ConversationBufferMemory( - memory_key="memory", - chat_memory=ChatMessageHistory( - client=self.client, key=f"history-{chat_id or 'default'}" - ), - return_messages=True, - input_key="input", - ) - - if len(memory.chat_memory.messages) == 0: - self.add_seed_chat(memory.chat_memory) - return memory - - def get_relevant_history(self, prompt: str): - relevant_docs = self.get_vectorstore().similarity_search(prompt, k=3) - return "\n".join( - [relevant_docs.page_content for relevant_docs in relevant_docs] - ) - - def get_tools(self, chat_id: str) -> List[Tool]: - return [ - SearchTool(self.client), - GenerateImageTool(self.client), - VideoMessageTool(self.client, voice_tool=self.voice_tool()), - ] - - def voice_tool(self) -> Optional[Tool]: - """Return tool to generate spoken version of output text.""" - if self.config.generate_voice_responses: - return GenerateSpeechTool( - client=self.client, - voice_id=self.config.elevenlabs_voice_id, - elevenlabs_api_key=self.config.elevenlabs_api_key, - ) - else: - return None - - @classmethod - def config_cls(cls) -> Type[Config]: - return CompanionConfig diff --git a/python/src/base.py b/python/src/base.py deleted file mode 100644 index 7f3b34f..0000000 --- a/python/src/base.py +++ /dev/null @@ -1,143 +0,0 @@ -import logging -import re -from abc import abstractmethod -from typing import List, Optional, Type - -from langchain.agents import Tool, AgentExecutor -from langchain.memory.chat_memory import BaseChatMemory -from steamship import Block -from steamship.agents.mixins.transports.steamship_widget import SteamshipWidgetTransport -from steamship.agents.mixins.transports.telegram import ( - TelegramTransportConfig, - TelegramTransport, -) -from steamship.agents.schema import ( - AgentContext, - Agent, -) -from steamship.agents.service.agent_service import AgentService -from steamship.invocable import post, Config, InvocationContext -from steamship.utils.kv_store import KeyValueStore - -from utils import is_uuid, UUID_PATTERN, replace_markdown_with_uuid - - -class ExtendedTelegramTransport(TelegramTransport): - def instance_init(self, config: Config, invocation_context: InvocationContext): - if config.bot_token: - self.api_root = f"{config.api_base}{config.bot_token}" - super().instance_init(config=config, invocation_context=invocation_context) - - -class LangChainTelegramBot(AgentService): - """Deployable Multimodal Agent that illustrates a character personality with voice. - - NOTE: To extend and deploy this agent, copy and paste the code into api.py. - """ - - USED_MIXIN_CLASSES = [ExtendedTelegramTransport, SteamshipWidgetTransport] - config: TelegramTransportConfig - - def __init__(self, **kwargs): - - super().__init__(**kwargs) - - # Set up bot_token - self.store = KeyValueStore(self.client, store_identifier="config") - bot_token = self.store.get("bot_token") - if bot_token: - bot_token = bot_token.get("token") - self.config.bot_token = self.config.bot_token or bot_token - - # Add transport mixins - self.add_mixin( - SteamshipWidgetTransport(client=self.client, agent_service=self, agent=None) - ) - self.add_mixin( - ExtendedTelegramTransport( - client=self.client, - config=self.config, - agent_service=self, - agent=None, - ), - permit_overwrite_of_existing_methods=True, - ) - - @post("connect_telegram", public=True) - def connect_telegram(self, bot_token: str): - self.store.set("bot_token", {"token": bot_token}) - self.config.bot_token = bot_token - - try: - self.instance_init() - return "OK" - except Exception as e: - return f"Could not set webhook for bot. Exception: {e}" - - @classmethod - def config_cls(cls) -> Type[Config]: - return TelegramTransportConfig - - @abstractmethod - def get_agent(self, chat_id: str, name: Optional[str] = None) -> AgentExecutor: - raise NotImplementedError() - - @abstractmethod - def get_memory(self, chat_id: str) -> BaseChatMemory: - raise NotImplementedError() - - @abstractmethod - def get_tools(self, chat_id: str) -> List[Tool]: - raise NotImplementedError() - - @abstractmethod - def get_relevant_history(self, prompt: str) -> str: - raise NotImplementedError() - - def voice_tool(self) -> Optional[Tool]: - return None - - def format_response(self, response): - response = replace_markdown_with_uuid(response) - response = UUID_PATTERN.split(response) - response = [re.sub(r"^\W+", "", el) for el in response] - response = [el for el in response if el] - if audio_tool := self.voice_tool(): - response_messages = [] - for message in response: - response_messages.append(message) - if not is_uuid(message): - audio_uuid = audio_tool.run(message) - response_messages.append(audio_uuid) - else: - response_messages = response - return [ - Block.get(self.client, _id=response) - if is_uuid(response) - else Block(text=response) - for response in response_messages - ] - - def run_agent( - self, - agent: Agent, - context: AgentContext, - name: Optional[str] = None, - ): - incoming_message = context.chat_history.last_user_message - chat_id = context.metadata.get("chat_id") or incoming_message.chat_id - - if incoming_message.text == "/new": - self.get_memory(chat_id).chat_memory.clear() - return [Block(text="New conversation started.")] - - response = self.get_agent(chat_id, name).run( - input=incoming_message.text, - relevantHistory=self.get_relevant_history(incoming_message.text), - ) - - output_messages = self.format_response(response) - - for func in context.emit_funcs: - logging.info(f"Emitting via function: {func.__name__}") - func(output_messages, context.metadata) From 704536dc410a4927393ce964295903b1d553bc5e Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Mon, 24 Jul 2023 22:03:53 +0200 Subject: [PATCH 29/31] Clean init companions --- python/init_companions.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/python/init_companions.py b/python/init_companions.py index 1a6f703..eb168c8 100644 --- a/python/init_companions.py +++ b/python/init_companions.py @@ -2,28 +2,18 @@ import re import sys from pathlib import Path -from uuid import uuid1 - -from steamship.cli.cli import deploy sys.path.append(str((Path(__file__) / ".." / "src").resolve())) import click from steamship import Steamship from steamship.cli.create_instance import load_manifest -from src.api import FileType - @click.command() @click.pass_context def init_companions(ctx): companions_dir = (Path(__file__) / ".." / ".." / "companions").resolve() - if click.confirm( - "Do you want to deploy a new version of your companion?", default=True - ): - ctx.invoke(deploy) - new_companions = {} for companion in companions_dir.iterdir(): if companion.suffix == ".txt": @@ -47,8 +37,7 @@ def init_companions(ctx): client = Steamship(workspace=companion.stem.lower()) manifest = load_manifest() instance = client.use( - manifest.handle, - version=manifest.version, + "ai-companion", config={ "name": companion.stem, "preamble": preamble.strip(), @@ -59,7 +48,7 @@ def init_companions(ctx): instance.invoke( "index_content", content=backstory.strip(), - file_type=FileType.TEXT, + file_type="TEXT", metadata={"title": "backstory"}, ) From bd93af55b44475e0901c985993871366c2b72b05 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Mon, 24 Jul 2023 22:05:07 +0200 Subject: [PATCH 30/31] Remove steamship.json --- python/steamship.json | 79 ------------------------------------------- 1 file changed, 79 deletions(-) delete mode 100644 python/steamship.json diff --git a/python/steamship.json b/python/steamship.json deleted file mode 100644 index 83eba44..0000000 --- a/python/steamship.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "type": "package", - "handle": "ai-companion", - "version": "0.0.2", - "description": "", - "author": "", - "entrypoint": "Unused", - "public": true, - "plugin": null, - "build_config": { - "ignore": [ - "tests", - "examples" - ] - }, - "configTemplate": { - "bot_token": { - "type": "string", - "description": "The secret token for your Telegram bot", - "default": "" - }, - "api_base": { - "type": "string", - "description": "The root API for Telegram", - "default": "https://api.telegram.org/bot" - }, - "name": { - "type": "string", - "description": "The name of your companion", - "default": null - }, - "preamble": { - "type": "string", - "description": "The preamble of your companion", - "default": null - }, - "seed_chat": { - "type": "string", - "description": "The seed chat of your companion", - "default": null - }, - "generate_voice_responses": { - "type": "boolean", - "description": "Enable voice responses", - "default": true - }, - "elevenlabs_api_key": { - "type": "string", - "description": "Optional API KEY for ElevenLabs Voice Bot", - "default": "" - }, - "elevenlabs_voice_id": { - "type": "string", - "description": "Optional voice_id for ElevenLabs Voice Bot", - "default": "" - } - }, - "steamshipRegistry": { - "tagline": "", - "tagline2": null, - "usefulFor": null, - "videoUrl": null, - "githubUrl": null, - "demoUrl": null, - "blogUrl": null, - "jupyterUrl": null, - "authorGithub": null, - "authorName": null, - "authorEmail": null, - "authorTwitter": null, - "authorUrl": null, - "tags": [ - "Telegram", - "LangChain", - "GPT", - "Companion" - ] - } -} \ No newline at end of file From fbd4cf4a94f488d528f2d7cb354096ac3667c477 Mon Sep 17 00:00:00 2001 From: Enias Cailliau Date: Tue, 25 Jul 2023 12:00:22 +0200 Subject: [PATCH 31/31] Fix init_companion to support single character updates --- python/init_companions.py | 93 +++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/python/init_companions.py b/python/init_companions.py index eb168c8..a4db278 100644 --- a/python/init_companions.py +++ b/python/init_companions.py @@ -3,60 +3,27 @@ import sys from pathlib import Path +from steamship.cli.cli import deploy + sys.path.append(str((Path(__file__) / ".." / "src").resolve())) import click from steamship import Steamship -from steamship.cli.create_instance import load_manifest @click.command() +@click.option('--all', default=False) @click.pass_context -def init_companions(ctx): +def init_companions(ctx, all: bool): companions_dir = (Path(__file__) / ".." / ".." / "companions").resolve() new_companions = {} - for companion in companions_dir.iterdir(): - if companion.suffix == ".txt": - companion_file = companion.open().read() - preamble, rest = companion_file.split("###ENDPREAMBLE###", 1) - seed_chat, backstory = rest.split("###ENDSEEDCHAT###", 1) - - pattern = r"### (.*?):(.*?)(?=###|$)" - - # Find all matches - matches = re.findall(pattern, seed_chat, re.DOTALL) - if matches: - seed_chat = [] - for match in matches: - user = match[0] - message = match[1].strip().replace("\\n\\n", "") - seed_chat.append(f"{user}:{message}") - seed_chat = "\n".join(seed_chat) - # Create instances for your companion - print(f"Creating an instance for {companion.stem}") - client = Steamship(workspace=companion.stem.lower()) - manifest = load_manifest() - instance = client.use( - "ai-companion", - config={ - "name": companion.stem, - "preamble": preamble.strip(), - "seed_chat": seed_chat.strip(), - }, - ) - - instance.invoke( - "index_content", - content=backstory.strip(), - file_type="TEXT", - metadata={"title": "backstory"}, - ) - - new_companions[companion.stem] = { - "name": companion.stem, - "llm": "steamship", - "generateEndpoint": f"{instance.invocation_url}answer", - } + if all: + for companion in companions_dir.iterdir(): + if companion.suffix == ".txt": + new_companions[companion.stem] = _init_companion(companion) + else: + companion_name = click.prompt("What's the name of your companion?") + new_companions[companion_name] = _init_companion(companions_dir / f"{companion_name}.txt") if click.confirm("Do you want to update the companions.json file?", default=True): companions = json.load((companions_dir / "companions.json").open()) @@ -72,5 +39,43 @@ def init_companions(ctx): ) +def _init_companion(companion): + companion_file = companion.open().read() + preamble, rest = companion_file.split("###ENDPREAMBLE###", 1) + seed_chat, backstory = rest.split("###ENDSEEDCHAT###", 1) + pattern = r"### (.*?):(.*?)(?=###|$)" + # Find all matches + matches = re.findall(pattern, seed_chat, re.DOTALL) + if matches: + seed_chat = [] + for match in matches: + user = match[0] + message = match[1].strip().replace("\\n\\n", "") + seed_chat.append(f"{user}:{message}") + seed_chat = "\n".join(seed_chat) + # Create instances for your companion + print(f"Creating an instance for {companion.stem}") + client = Steamship(workspace=f"{companion.stem.lower()}_workspace_new") + instance = client.use( + "ai-companion", + config={ + "name": companion.stem, + "preamble": preamble.strip(), + "seed_chat": seed_chat.strip(), + }, + ) + instance.invoke( + "index_content", + content=backstory.strip(), + file_type="TEXT", + metadata={"title": "backstory"}, + ) + return { + "name": companion.stem, + "llm": "steamship", + "generateEndpoint": f"{instance.invocation_url}answer", + } + + if __name__ == "__main__": init_companions()