From 7f74dba5d63ba8e0e588cb33461494e19e7d7541 Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Mon, 8 Jan 2024 10:25:46 +0100 Subject: [PATCH 1/9] Remove caching of LLM instances --- app/config.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/config.py b/app/config.py index 63a2414a..f596d58d 100644 --- a/app/config.py +++ b/app/config.py @@ -30,10 +30,11 @@ def check_type(cls, v): return v def get_instance(cls): - if cls.instance is not None: - return cls.instance - cls.instance = OpenAI(**cls.llm_credentials) - return cls.instance + return OpenAI(**cls.llm_credentials) + # if cls.instance is not None: + # return cls.instance + # cls.instance = OpenAI(**cls.llm_credentials) + # return cls.instance class StrategyLLMConfig(LLMModelConfig): @@ -47,13 +48,16 @@ def check_type(cls, v): return v def get_instance(cls): - if cls.instance is not None: - return cls.instance - # Local import needed to avoid circular dependency from app.llms.strategy_llm import StrategyLLM - cls.instance = StrategyLLM(cls.llms) - return cls.instance + return StrategyLLM(cls.llms) + # if# cls.instance is not None: + # return cls.instance + # # Local import needed to avoid circular dependency + # from app.llms.strategy_llm import StrategyLLM + # + # cls.instance = StrategyLLM(cls.llms) + # return cls.instance class APIKeyConfig(BaseModel): From 413e45b5e29430fabece838a6509f51be48489b1 Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Mon, 8 Jan 2024 11:19:19 +0100 Subject: [PATCH 2/9] Further small changes --- docker/pyris-production.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/pyris-production.yml b/docker/pyris-production.yml index c246fab1..dc74e13b 100644 --- a/docker/pyris-production.yml +++ b/docker/pyris-production.yml @@ -15,8 +15,11 @@ services: condition: service_started pull_policy: always restart: always + environment: + OPENAI_LOG: "debug" volumes: - ${PYRIS_APPLICATION_YML_FILE:-../application.example.yml}:/app/application.yml:ro + command: "poetry run uvicorn app.main:app --host '0.0.0.0'" networks: - pyris From e0400e7f58b69be78b567c7f99651117cbc5961d Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Mon, 8 Jan 2024 13:12:31 +0100 Subject: [PATCH 3/9] Try health check lock --- app/services/circuit_breaker.py | 48 +++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/app/services/circuit_breaker.py b/app/services/circuit_breaker.py index 077db14f..d5b1e342 100644 --- a/app/services/circuit_breaker.py +++ b/app/services/circuit_breaker.py @@ -1,6 +1,9 @@ import logging from enum import Enum from app.services.cache import cache_store +from threading import Lock + +health_locks = {} log = logging.getLogger(__name__) @@ -109,23 +112,28 @@ def get_status(cls, checkhealth_func, cache_key: str): Returns: Status of the cache_key. Circuit is CLOSED if cache_key is up and running, and OPEN otherwise. """ - status_key = f"{cache_key}:status" - status = cache_store.get(status_key) - - if status: - return cls.Status(status) - - is_up = False - try: - is_up = checkhealth_func() - except Exception as e: - log.error(e) - - if is_up: - cache_store.set( - status_key, cls.Status.CLOSED, ex=cls.CLOSED_TTL_SECONDS - ) - return cls.Status.CLOSED - - cache_store.set(status_key, cls.Status.OPEN, ex=cls.OPEN_TTL_SECONDS) - return cls.Status.OPEN + if cache_key not in health_locks: + health_locks[cache_key] = Lock() + + # Acquire the lock + with health_locks[cache_key]: + status_key = f"{cache_key}:status" + status = cache_store.get(status_key) + + if status: + return cls.Status(status) + + is_up = False + try: + is_up = checkhealth_func() + except Exception as e: + log.error(e) + + if is_up: + cache_store.set( + status_key, cls.Status.CLOSED, ex=cls.CLOSED_TTL_SECONDS + ) + return cls.Status.CLOSED + + cache_store.set(status_key, cls.Status.OPEN, ex=cls.OPEN_TTL_SECONDS) + return cls.Status.OPEN From 5a5c3e0a7603a6e8a26189322ce8a439a17369e6 Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Mon, 8 Jan 2024 13:46:44 +0100 Subject: [PATCH 4/9] Add more logging --- app/config.py | 18 ++++++++++++++++++ app/services/circuit_breaker.py | 4 +++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index f596d58d..24c2b8ec 100644 --- a/app/config.py +++ b/app/config.py @@ -1,9 +1,27 @@ import os +import types from guidance.llms import OpenAI +import guidance.llms._openai from pyaml_env import parse_config from pydantic import BaseModel, validator, Field, typing +old_add_text_to_chat_mode = guidance.llms._openai.add_text_to_chat_mode + + +def new_add_text_to_chat_mode(chat_mode): + if isinstance(chat_mode, (types.AsyncGeneratorType, types.GeneratorType)): + print("new_add_text_to_chat_mode", chat_mode) + return guidance.llms._openai.add_text_to_chat_mode_generator(chat_mode) + else: + print("new_add_text_to_chat_mode", chat_mode.items()) + for c in chat_mode["choices"]: + c["text"] = c["message"]["content"] + return chat_mode + + +guidance.llms._openai.add_text_to_chat_mode = new_add_text_to_chat_mode + class LLMModelSpecs(BaseModel): context_length: int diff --git a/app/services/circuit_breaker.py b/app/services/circuit_breaker.py index d5b1e342..fb080c1c 100644 --- a/app/services/circuit_breaker.py +++ b/app/services/circuit_breaker.py @@ -135,5 +135,7 @@ def get_status(cls, checkhealth_func, cache_key: str): ) return cls.Status.CLOSED - cache_store.set(status_key, cls.Status.OPEN, ex=cls.OPEN_TTL_SECONDS) + cache_store.set( + status_key, cls.Status.OPEN, ex=cls.OPEN_TTL_SECONDS + ) return cls.Status.OPEN From 8298e7810af12c8b911cabb217ccc5b0174ed578 Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Mon, 8 Jan 2024 14:05:47 +0100 Subject: [PATCH 5/9] Add if check to problematic place --- app/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 24c2b8ec..ec653b0c 100644 --- a/app/config.py +++ b/app/config.py @@ -16,7 +16,10 @@ def new_add_text_to_chat_mode(chat_mode): else: print("new_add_text_to_chat_mode", chat_mode.items()) for c in chat_mode["choices"]: - c["text"] = c["message"]["content"] + if "message" in c and "content" in c["message"]: + c["text"] = c["message"]["content"] + else: + c["text"] = "" return chat_mode From 9cc5143d627e392e9c5a23a24d41b77deb0f1854 Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Mon, 8 Jan 2024 14:11:22 +0100 Subject: [PATCH 6/9] "" -> None --- app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index ec653b0c..6094012d 100644 --- a/app/config.py +++ b/app/config.py @@ -19,7 +19,7 @@ def new_add_text_to_chat_mode(chat_mode): if "message" in c and "content" in c["message"]: c["text"] = c["message"]["content"] else: - c["text"] = "" + c["text"] = None return chat_mode From 88f71fe7d50ef0bb6410306c56c8bccf3857cc8b Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Mon, 8 Jan 2024 14:17:56 +0100 Subject: [PATCH 7/9] None -> " " --- app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/config.py b/app/config.py index 6094012d..6892d0a4 100644 --- a/app/config.py +++ b/app/config.py @@ -19,7 +19,7 @@ def new_add_text_to_chat_mode(chat_mode): if "message" in c and "content" in c["message"]: c["text"] = c["message"]["content"] else: - c["text"] = None + c["text"] = " " return chat_mode From 82b759ae4bbe8e9967f729b9a1633a4dbe015613 Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Mon, 8 Jan 2024 14:34:48 +0100 Subject: [PATCH 8/9] Cleanup --- app/config.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/app/config.py b/app/config.py index 6892d0a4..2bb7bfc8 100644 --- a/app/config.py +++ b/app/config.py @@ -4,17 +4,15 @@ from guidance.llms import OpenAI import guidance.llms._openai from pyaml_env import parse_config -from pydantic import BaseModel, validator, Field, typing +from pydantic import BaseModel, validator old_add_text_to_chat_mode = guidance.llms._openai.add_text_to_chat_mode def new_add_text_to_chat_mode(chat_mode): if isinstance(chat_mode, (types.AsyncGeneratorType, types.GeneratorType)): - print("new_add_text_to_chat_mode", chat_mode) return guidance.llms._openai.add_text_to_chat_mode_generator(chat_mode) else: - print("new_add_text_to_chat_mode", chat_mode.items()) for c in chat_mode["choices"]: if "message" in c and "content" in c["message"]: c["text"] = c["message"]["content"] @@ -42,7 +40,6 @@ def get_instance(cls): class OpenAIConfig(LLMModelConfig): spec: LLMModelSpecs llm_credentials: dict - instance: typing.Any = Field(repr=False) @validator("type") def check_type(cls, v): @@ -52,15 +49,10 @@ def check_type(cls, v): def get_instance(cls): return OpenAI(**cls.llm_credentials) - # if cls.instance is not None: - # return cls.instance - # cls.instance = OpenAI(**cls.llm_credentials) - # return cls.instance class StrategyLLMConfig(LLMModelConfig): llms: list[str] - instance: typing.Any = Field(repr=False) @validator("type") def check_type(cls, v): @@ -72,13 +64,6 @@ def get_instance(cls): from app.llms.strategy_llm import StrategyLLM return StrategyLLM(cls.llms) - # if# cls.instance is not None: - # return cls.instance - # # Local import needed to avoid circular dependency - # from app.llms.strategy_llm import StrategyLLM - # - # cls.instance = StrategyLLM(cls.llms) - # return cls.instance class APIKeyConfig(BaseModel): From 7633380ec1d0837defa25ae3d84d3db20915e27a Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Mon, 8 Jan 2024 15:58:48 +0100 Subject: [PATCH 9/9] Tryfixing again... --- app/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/config.py b/app/config.py index 2bb7bfc8..684643a1 100644 --- a/app/config.py +++ b/app/config.py @@ -1,6 +1,6 @@ import os import types - +import copy from guidance.llms import OpenAI import guidance.llms._openai from pyaml_env import parse_config @@ -48,7 +48,9 @@ def check_type(cls, v): return v def get_instance(cls): - return OpenAI(**cls.llm_credentials) + print("Specs", cls.llm_credentials) + llm_credentials = copy.deepcopy(cls.llm_credentials) + return OpenAI(**llm_credentials) class StrategyLLMConfig(LLMModelConfig):