From 5414ed4f5b34af0604fc6772224f0f1c2caaa693 Mon Sep 17 00:00:00 2001 From: monst Date: Mon, 9 Oct 2023 17:21:16 +0200 Subject: [PATCH 01/15] Initial commit --- app/models/dtos.py | 16 +++------- app/routes/messages.py | 9 ++---- app/services/guidance_wrapper.py | 11 +++++-- tests/routes/messages_test.py | 12 ++++---- tests/services/guidance_wrapper_test.py | 41 +++++-------------------- 5 files changed, 27 insertions(+), 62 deletions(-) diff --git a/app/models/dtos.py b/app/models/dtos.py index 2fe7c28f..55eedcde 100644 --- a/app/models/dtos.py +++ b/app/models/dtos.py @@ -13,11 +13,6 @@ class ContentType(str, Enum): TEXT = "text" -class Content(BaseModel): - text_content: str = Field(..., alias="textContent") - type: ContentType - - class SendMessageRequest(BaseModel): class Template(BaseModel): id: int @@ -29,14 +24,11 @@ class Template(BaseModel): class SendMessageResponse(BaseModel): - class Message(BaseModel): - sent_at: datetime = Field( - alias="sentAt", default_factory=datetime.utcnow - ) - content: list[Content] - used_model: str = Field(..., alias="usedModel") - message: Message + sent_at: datetime = Field( + alias="sentAt", default_factory=datetime.utcnow + ) + content: dict class ModelStatus(BaseModel): diff --git a/app/routes/messages.py b/app/routes/messages.py index ee8e98d5..4dc873d1 100644 --- a/app/routes/messages.py +++ b/app/routes/messages.py @@ -52,13 +52,8 @@ def send_message(body: SendMessageRequest) -> SendMessageResponse: except Exception as e: raise InternalServerException(str(e)) - # Turn content into an array if it's not already - if not isinstance(content, list): - content = [content] - return SendMessageResponse( usedModel=body.preferred_model, - message=SendMessageResponse.Message( - sentAt=datetime.now(timezone.utc), content=content - ), + sentAt=datetime.now(timezone.utc), + content=content, ) diff --git a/app/services/guidance_wrapper.py b/app/services/guidance_wrapper.py index 34b88802..3ab4ff0f 100644 --- a/app/services/guidance_wrapper.py +++ b/app/services/guidance_wrapper.py @@ -32,6 +32,10 @@ def query(self) -> Content: ValueError: if handlebars do not generate 'response' """ + import re + pattern = r'{{(?:gen|geneach|set) [\'"]([^\'"]+)[\'"]$}}' + var_names = re.findall(pattern, input_string) + template = guidance(self.handlebars) result = template( llm=self._get_llm(), @@ -42,10 +46,11 @@ def query(self) -> Content: if isinstance(result._exception, Exception): raise result._exception - if "response" not in result: - raise ValueError("The handlebars do not generate 'response'") + generated_vars = { + var_name: result[var_name] for var_name in var_names if var_name in result + } - return Content(type=ContentType.TEXT, textContent=result["response"]) + return generated_vars def is_up(self) -> bool: """Check if the chosen LLM model is up. diff --git a/tests/routes/messages_test.py b/tests/routes/messages_test.py index e0f09786..725514cf 100644 --- a/tests/routes/messages_test.py +++ b/tests/routes/messages_test.py @@ -31,9 +31,9 @@ def test_send_message(test_client, headers, mocker): mocker.patch.object( GuidanceWrapper, "query", - return_value=Content( - type=ContentType.TEXT, textContent="some content" - ), + return_value={ + "response": "some content", + }, autospec=True, ) @@ -55,9 +55,9 @@ def test_send_message(test_client, headers, mocker): assert response.status_code == 200 assert response.json() == { "usedModel": "GPT35_TURBO", - "message": { - "sentAt": "2023-06-16T01:21:34+00:00", - "content": [{"textContent": "some content", "type": "text"}], + "sentAt": "2023-06-16T01:21:34+00:00", + "content": { + "response": "some content", }, } diff --git a/tests/services/guidance_wrapper_test.py b/tests/services/guidance_wrapper_test.py index 502da4ba..cb0f540c 100644 --- a/tests/services/guidance_wrapper_test.py +++ b/tests/services/guidance_wrapper_test.py @@ -1,7 +1,6 @@ import pytest import guidance -from app.models.dtos import Content, ContentType from app.services.guidance_wrapper import GuidanceWrapper from app.config import OpenAIConfig @@ -33,9 +32,8 @@ def test_query_success(mocker): result = guidance_wrapper.query() - assert isinstance(result, Content) - assert result.type == ContentType.TEXT - assert result.text_content == "the output" + assert isinstance(result, dict) + assert result['response'] == "the output" def test_query_using_truncate_function(mocker): @@ -59,9 +57,9 @@ def test_query_using_truncate_function(mocker): result = guidance_wrapper.query() - assert isinstance(result, Content) - assert result.type == ContentType.TEXT - assert result.text_content == "the" + assert isinstance(result, dict) + assert result['answer'] == "the output" + assert result['response'] == "the" def test_query_missing_required_params(mocker): @@ -84,30 +82,5 @@ def test_query_missing_required_params(mocker): with pytest.raises(KeyError, match="Command/variable 'query' not found!"): result = guidance_wrapper.query() - assert isinstance(result, Content) - assert result.type == ContentType.TEXT - assert result.text_content == "the output" - - -def test_query_handlebars_not_generate_response(mocker): - mocker.patch.object( - GuidanceWrapper, - "_get_llm", - return_value=guidance.llms.Mock("the output"), - ) - - handlebars = "Not a valid handlebars" - guidance_wrapper = GuidanceWrapper( - model=llm_model_config, - handlebars=handlebars, - parameters={"query": "Something"}, - ) - - with pytest.raises( - ValueError, match="The handlebars do not generate 'response'" - ): - result = guidance_wrapper.query() - - assert isinstance(result, Content) - assert result.type == ContentType.TEXT - assert result.text_content == "the output" + assert isinstance(result, dict) + assert result['response'] == "the output" From 153d2b78caed6656cfab779bdd5a2afeb9383555 Mon Sep 17 00:00:00 2001 From: monst Date: Mon, 9 Oct 2023 17:31:33 +0200 Subject: [PATCH 02/15] Reformat code --- app/models/dtos.py | 4 +--- app/services/guidance_wrapper.py | 12 +++++++----- tests/services/guidance_wrapper_test.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/models/dtos.py b/app/models/dtos.py index 55eedcde..d52d0487 100644 --- a/app/models/dtos.py +++ b/app/models/dtos.py @@ -25,9 +25,7 @@ class Template(BaseModel): class SendMessageResponse(BaseModel): used_model: str = Field(..., alias="usedModel") - sent_at: datetime = Field( - alias="sentAt", default_factory=datetime.utcnow - ) + sent_at: datetime = Field(alias="sentAt", default_factory=datetime.utcnow) content: dict diff --git a/app/services/guidance_wrapper.py b/app/services/guidance_wrapper.py index 3ab4ff0f..82160189 100644 --- a/app/services/guidance_wrapper.py +++ b/app/services/guidance_wrapper.py @@ -1,7 +1,6 @@ import guidance from app.config import LLMModelConfig -from app.models.dtos import Content, ContentType from app.services.guidance_functions import truncate @@ -21,7 +20,7 @@ def __init__( self.handlebars = handlebars self.parameters = parameters - def query(self) -> Content: + def query(self) -> dict: """Get response from a chosen LLM model. Returns: @@ -33,8 +32,9 @@ def query(self) -> Content: """ import re + pattern = r'{{(?:gen|geneach|set) [\'"]([^\'"]+)[\'"]$}}' - var_names = re.findall(pattern, input_string) + var_names = re.findall(pattern, self.handlebars) template = guidance(self.handlebars) result = template( @@ -47,7 +47,9 @@ def query(self) -> Content: raise result._exception generated_vars = { - var_name: result[var_name] for var_name in var_names if var_name in result + var_name: result[var_name] + for var_name in var_names + if var_name in result } return generated_vars @@ -69,7 +71,7 @@ def is_up(self) -> bool: content = ( GuidanceWrapper(model=self.model, handlebars=handlebars) .query() - .text_content + .get("response") ) return content == "1" diff --git a/tests/services/guidance_wrapper_test.py b/tests/services/guidance_wrapper_test.py index cb0f540c..84fe9333 100644 --- a/tests/services/guidance_wrapper_test.py +++ b/tests/services/guidance_wrapper_test.py @@ -33,7 +33,7 @@ def test_query_success(mocker): result = guidance_wrapper.query() assert isinstance(result, dict) - assert result['response'] == "the output" + assert result["response"] == "the output" def test_query_using_truncate_function(mocker): @@ -58,8 +58,8 @@ def test_query_using_truncate_function(mocker): result = guidance_wrapper.query() assert isinstance(result, dict) - assert result['answer'] == "the output" - assert result['response'] == "the" + assert result["answer"] == "the output" + assert result["response"] == "the" def test_query_missing_required_params(mocker): @@ -83,4 +83,4 @@ def test_query_missing_required_params(mocker): result = guidance_wrapper.query() assert isinstance(result, dict) - assert result['response'] == "the output" + assert result["response"] == "the output" From 28961f0d2c6d5cbb25e7eed6c5ee3ec884b031fd Mon Sep 17 00:00:00 2001 From: monst Date: Tue, 10 Oct 2023 18:56:20 +0200 Subject: [PATCH 03/15] Document regex --- app/services/guidance_wrapper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/services/guidance_wrapper.py b/app/services/guidance_wrapper.py index 82160189..d1eb9bce 100644 --- a/app/services/guidance_wrapper.py +++ b/app/services/guidance_wrapper.py @@ -31,8 +31,11 @@ def query(self) -> dict: ValueError: if handlebars do not generate 'response' """ + # Perform a regex search to find the names of the variables being generated + # by the handlebars template + # This regex matches strings like: {{gen 'response' temperature=0.0 max_tokens=500}} + # and extracts the variable name 'response' import re - pattern = r'{{(?:gen|geneach|set) [\'"]([^\'"]+)[\'"]$}}' var_names = re.findall(pattern, self.handlebars) From c7c3cad695e922ae0d8beb24a3f882ae4b573bea Mon Sep 17 00:00:00 2001 From: monst Date: Thu, 12 Oct 2023 11:42:19 +0200 Subject: [PATCH 04/15] Fix regex --- app/services/guidance_wrapper.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/services/guidance_wrapper.py b/app/services/guidance_wrapper.py index d1eb9bce..baf3a29d 100644 --- a/app/services/guidance_wrapper.py +++ b/app/services/guidance_wrapper.py @@ -1,4 +1,5 @@ import guidance +import re from app.config import LLMModelConfig from app.services.guidance_functions import truncate @@ -28,15 +29,13 @@ def query(self) -> dict: Raises: Reraises exception from guidance package - ValueError: if handlebars do not generate 'response' """ # Perform a regex search to find the names of the variables being generated # by the handlebars template # This regex matches strings like: {{gen 'response' temperature=0.0 max_tokens=500}} # and extracts the variable name 'response' - import re - pattern = r'{{(?:gen|geneach|set) [\'"]([^\'"]+)[\'"]$}}' + pattern = r'{{#?(?:gen|geneach|set) +[\'"]([^\'"]+)[\'"]' var_names = re.findall(pattern, self.handlebars) template = guidance(self.handlebars) From 394de2e3203ee994857d0ba8cf0955007450b5db Mon Sep 17 00:00:00 2001 From: monst Date: Thu, 12 Oct 2023 11:59:28 +0200 Subject: [PATCH 05/15] Restore V1 functionality, add V2 endpoint --- app/models/dtos.py | 18 +++++++++++++++ app/routes/messages.py | 50 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/app/models/dtos.py b/app/models/dtos.py index d52d0487..b1ffc8a2 100644 --- a/app/models/dtos.py +++ b/app/models/dtos.py @@ -13,6 +13,12 @@ class ContentType(str, Enum): TEXT = "text" +# V1 API only +class Content(BaseModel): + text_content: str = Field(..., alias="textContent") + type: ContentType + + class SendMessageRequest(BaseModel): class Template(BaseModel): id: int @@ -23,7 +29,19 @@ class Template(BaseModel): parameters: dict +# V1 API only class SendMessageResponse(BaseModel): + class Message(BaseModel): + sent_at: datetime = Field( + alias="sentAt", default_factory=datetime.utcnow + ) + content: list[Content] + + used_model: str = Field(..., alias="usedModel") + message: Message + + +class SendMessageResponseV2(BaseModel): used_model: str = Field(..., alias="usedModel") sent_at: datetime = Field(alias="sentAt", default_factory=datetime.utcnow) content: dict diff --git a/app/routes/messages.py b/app/routes/messages.py index 4dc873d1..383091a3 100644 --- a/app/routes/messages.py +++ b/app/routes/messages.py @@ -11,7 +11,13 @@ InvalidModelException, ) from app.dependencies import TokenPermissionsValidator -from app.models.dtos import SendMessageRequest, SendMessageResponse +from app.models.dtos import ( + SendMessageRequest, + SendMessageResponse, + Content, + ContentType, + SendMessageResponseV2, +) from app.services.circuit_breaker import CircuitBreaker from app.services.guidance_wrapper import GuidanceWrapper from app.config import settings @@ -19,10 +25,7 @@ router = APIRouter(tags=["messages"]) -@router.post( - "/api/v1/messages", dependencies=[Depends(TokenPermissionsValidator())] -) -def send_message(body: SendMessageRequest) -> SendMessageResponse: +def execute_call(body: SendMessageRequest) -> dict: try: model = settings.pyris.llms[body.preferred_model] except ValueError as e: @@ -35,7 +38,7 @@ def send_message(body: SendMessageRequest) -> SendMessageResponse: ) try: - content = CircuitBreaker.protected_call( + return CircuitBreaker.protected_call( func=guidance.query, cache_key=body.preferred_model, accepted_exceptions=( @@ -52,8 +55,41 @@ def send_message(body: SendMessageRequest) -> SendMessageResponse: except Exception as e: raise InternalServerException(str(e)) + +@router.post( + "/api/v1/messages", dependencies=[Depends(TokenPermissionsValidator())] +) +def send_message(body: SendMessageRequest) -> SendMessageResponse: + generated_vars = execute_call(body) + + # Restore the old behavior of only returning the 'response' variable for the v1 API + if "response" not in generated_vars: + raise InternalServerException( + str(ValueError("The handlebars do not generate 'response'")) + ) + return SendMessageResponse( + usedModel=body.preferred_model, + message=SendMessageResponse.Message( + sentAt=datetime.now(timezone.utc), + content=[ + Content( + type=ContentType.TEXT, + textContent=generated_vars["response"], + ) + ], + ), + ) + + +@router.post( + "/api/v2/messages", dependencies=[Depends(TokenPermissionsValidator())] +) +def send_message_v2(body: SendMessageRequest) -> SendMessageResponseV2: + generated_vars = execute_call(body) + + return SendMessageResponseV2( usedModel=body.preferred_model, sentAt=datetime.now(timezone.utc), - content=content, + content=generated_vars, ) From f6a9fb32422f5e394628e57bf96a0be1ab30b87d Mon Sep 17 00:00:00 2001 From: monst Date: Thu, 12 Oct 2023 12:38:25 +0200 Subject: [PATCH 06/15] Add documentation --- app/routes/messages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/routes/messages.py b/app/routes/messages.py index 383091a3..08f28d56 100644 --- a/app/routes/messages.py +++ b/app/routes/messages.py @@ -62,7 +62,7 @@ def execute_call(body: SendMessageRequest) -> dict: def send_message(body: SendMessageRequest) -> SendMessageResponse: generated_vars = execute_call(body) - # Restore the old behavior of only returning the 'response' variable for the v1 API + # Restore the old behavior of throwing an exception if no 'response' variable was generated if "response" not in generated_vars: raise InternalServerException( str(ValueError("The handlebars do not generate 'response'")) @@ -75,7 +75,7 @@ def send_message(body: SendMessageRequest) -> SendMessageResponse: content=[ Content( type=ContentType.TEXT, - textContent=generated_vars["response"], + textContent=generated_vars["response"], # V1 behavior: only return the 'response' variable ) ], ), @@ -91,5 +91,5 @@ def send_message_v2(body: SendMessageRequest) -> SendMessageResponseV2: return SendMessageResponseV2( usedModel=body.preferred_model, sentAt=datetime.now(timezone.utc), - content=generated_vars, + content=generated_vars, # V2 behavior: return all variables generated in the program ) From 581377fc4b802a4c274c1bc824e8e347dd50a606 Mon Sep 17 00:00:00 2001 From: monst Date: Thu, 12 Oct 2023 12:44:11 +0200 Subject: [PATCH 07/15] Update docs --- app/services/guidance_wrapper.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/services/guidance_wrapper.py b/app/services/guidance_wrapper.py index baf3a29d..834a8fd3 100644 --- a/app/services/guidance_wrapper.py +++ b/app/services/guidance_wrapper.py @@ -31,10 +31,12 @@ def query(self) -> dict: Reraises exception from guidance package """ - # Perform a regex search to find the names of the variables being generated - # by the handlebars template - # This regex matches strings like: {{gen 'response' temperature=0.0 max_tokens=500}} - # and extracts the variable name 'response' + # Perform a regex search to find the names of the variables + # being generated in the program. This regex matches strings like: + # {{gen 'response' temperature=0.0 max_tokens=500}} + # {{#geneach 'values' num_iterations=3}} + # {{set 'answer' (truncate response 3)}} + # and extracts the variable names 'response', 'values', and 'answer' pattern = r'{{#?(?:gen|geneach|set) +[\'"]([^\'"]+)[\'"]' var_names = re.findall(pattern, self.handlebars) From ebf96dc90b06208a81dcfb27517010eec8d147c4 Mon Sep 17 00:00:00 2001 From: monst Date: Thu, 12 Oct 2023 12:44:11 +0200 Subject: [PATCH 08/15] Update docs --- app/services/guidance_wrapper.py | 10 ++++++---- tests/routes/messages_test.py | 1 - 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/services/guidance_wrapper.py b/app/services/guidance_wrapper.py index baf3a29d..834a8fd3 100644 --- a/app/services/guidance_wrapper.py +++ b/app/services/guidance_wrapper.py @@ -31,10 +31,12 @@ def query(self) -> dict: Reraises exception from guidance package """ - # Perform a regex search to find the names of the variables being generated - # by the handlebars template - # This regex matches strings like: {{gen 'response' temperature=0.0 max_tokens=500}} - # and extracts the variable name 'response' + # Perform a regex search to find the names of the variables + # being generated in the program. This regex matches strings like: + # {{gen 'response' temperature=0.0 max_tokens=500}} + # {{#geneach 'values' num_iterations=3}} + # {{set 'answer' (truncate response 3)}} + # and extracts the variable names 'response', 'values', and 'answer' pattern = r'{{#?(?:gen|geneach|set) +[\'"]([^\'"]+)[\'"]' var_names = re.findall(pattern, self.handlebars) diff --git a/tests/routes/messages_test.py b/tests/routes/messages_test.py index 725514cf..a9b77ce5 100644 --- a/tests/routes/messages_test.py +++ b/tests/routes/messages_test.py @@ -1,6 +1,5 @@ import pytest from freezegun import freeze_time -from app.models.dtos import Content, ContentType from app.services.guidance_wrapper import GuidanceWrapper import app.config as config From 877119544d2eae00779450370c68994290bdba1a Mon Sep 17 00:00:00 2001 From: monst Date: Thu, 12 Oct 2023 12:52:17 +0200 Subject: [PATCH 09/15] Add message test for v2 api --- tests/routes/messages_test.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/routes/messages_test.py b/tests/routes/messages_test.py index a9b77ce5..b3b98368 100644 --- a/tests/routes/messages_test.py +++ b/tests/routes/messages_test.py @@ -50,9 +50,19 @@ def test_send_message(test_client, headers, mocker): "query": "Some query", }, } - response = test_client.post("/api/v1/messages", headers=headers, json=body) - assert response.status_code == 200 - assert response.json() == { + response_v1 = test_client.post("/api/v1/messages", headers=headers, json=body) + assert response_v1.status_code == 200 + assert response_v1.json() == { + "usedModel": "GPT35_TURBO", + "message": { + "sentAt": "2023-06-16T01:21:34+00:00", + "content": [{"textContent": "some content", "type": "text"}], + }, + } + + response_v2 = test_client.post("/api/v2/messages", headers=headers, json=body) + assert response_v2.status_code == 200 + assert response_v2.json() == { "usedModel": "GPT35_TURBO", "sentAt": "2023-06-16T01:21:34+00:00", "content": { From 63126fc25f19ef060b99f498d36136e0265de588 Mon Sep 17 00:00:00 2001 From: monst Date: Fri, 13 Oct 2023 14:12:09 +0200 Subject: [PATCH 10/15] Run formatter --- app/routes/messages.py | 4 +++- tests/routes/messages_test.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/routes/messages.py b/app/routes/messages.py index 08f28d56..46f03c89 100644 --- a/app/routes/messages.py +++ b/app/routes/messages.py @@ -75,7 +75,9 @@ def send_message(body: SendMessageRequest) -> SendMessageResponse: content=[ Content( type=ContentType.TEXT, - textContent=generated_vars["response"], # V1 behavior: only return the 'response' variable + textContent=generated_vars[ + "response" + ], # V1 behavior: only return the 'response' variable ) ], ), diff --git a/tests/routes/messages_test.py b/tests/routes/messages_test.py index b3b98368..3d5ba218 100644 --- a/tests/routes/messages_test.py +++ b/tests/routes/messages_test.py @@ -50,7 +50,9 @@ def test_send_message(test_client, headers, mocker): "query": "Some query", }, } - response_v1 = test_client.post("/api/v1/messages", headers=headers, json=body) + response_v1 = test_client.post( + "/api/v1/messages", headers=headers, json=body + ) assert response_v1.status_code == 200 assert response_v1.json() == { "usedModel": "GPT35_TURBO", @@ -60,7 +62,9 @@ def test_send_message(test_client, headers, mocker): }, } - response_v2 = test_client.post("/api/v2/messages", headers=headers, json=body) + response_v2 = test_client.post( + "/api/v2/messages", headers=headers, json=body + ) assert response_v2.status_code == 200 assert response_v2.json() == { "usedModel": "GPT35_TURBO", From d13bf7e4ec8921b31f1a513a5c90a859d2bfd48b Mon Sep 17 00:00:00 2001 From: monst Date: Fri, 13 Oct 2023 14:16:31 +0200 Subject: [PATCH 11/15] Shorten long lines --- app/routes/messages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/routes/messages.py b/app/routes/messages.py index 46f03c89..baa65d70 100644 --- a/app/routes/messages.py +++ b/app/routes/messages.py @@ -62,7 +62,7 @@ def execute_call(body: SendMessageRequest) -> dict: def send_message(body: SendMessageRequest) -> SendMessageResponse: generated_vars = execute_call(body) - # Restore the old behavior of throwing an exception if no 'response' variable was generated + # V1: Throw an exception if no 'response' variable was generated if "response" not in generated_vars: raise InternalServerException( str(ValueError("The handlebars do not generate 'response'")) @@ -77,7 +77,7 @@ def send_message(body: SendMessageRequest) -> SendMessageResponse: type=ContentType.TEXT, textContent=generated_vars[ "response" - ], # V1 behavior: only return the 'response' variable + ], # V1: only return the 'response' variable ) ], ), @@ -93,5 +93,5 @@ def send_message_v2(body: SendMessageRequest) -> SendMessageResponseV2: return SendMessageResponseV2( usedModel=body.preferred_model, sentAt=datetime.now(timezone.utc), - content=generated_vars, # V2 behavior: return all variables generated in the program + content=generated_vars, # V2: return all generated variables ) From 67722af9cd2f44a669962a8ae92e84cee1501851 Mon Sep 17 00:00:00 2001 From: Timor Morrien Date: Mon, 16 Oct 2023 16:04:06 +0200 Subject: [PATCH 12/15] Fixes after debugging session --- app/models/dtos.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/models/dtos.py b/app/models/dtos.py index b1ffc8a2..773b6af9 100644 --- a/app/models/dtos.py +++ b/app/models/dtos.py @@ -21,7 +21,6 @@ class Content(BaseModel): class SendMessageRequest(BaseModel): class Template(BaseModel): - id: int content: str template: Template From 8c683508189610e0d6b5351c2070050835c5d76f Mon Sep 17 00:00:00 2001 From: monst Date: Thu, 19 Oct 2023 09:33:00 +0200 Subject: [PATCH 13/15] Add V2 request type with only string as template --- app/models/dtos.py | 6 ++++++ app/routes/messages.py | 17 +++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/app/models/dtos.py b/app/models/dtos.py index 773b6af9..8dba80a6 100644 --- a/app/models/dtos.py +++ b/app/models/dtos.py @@ -40,6 +40,12 @@ class Message(BaseModel): message: Message +class SendMessageRequestV2(BaseModel): + template: str + preferred_model: str = Field(..., alias="preferredModel") + parameters: dict + + class SendMessageResponseV2(BaseModel): used_model: str = Field(..., alias="usedModel") sent_at: datetime = Field(alias="sentAt", default_factory=datetime.utcnow) diff --git a/app/routes/messages.py b/app/routes/messages.py index baa65d70..a324ed40 100644 --- a/app/routes/messages.py +++ b/app/routes/messages.py @@ -16,6 +16,7 @@ SendMessageResponse, Content, ContentType, + SendMessageRequestV2, SendMessageResponseV2, ) from app.services.circuit_breaker import CircuitBreaker @@ -25,22 +26,22 @@ router = APIRouter(tags=["messages"]) -def execute_call(body: SendMessageRequest) -> dict: +def execute_call(template, preferred_model, parameters) -> dict: try: - model = settings.pyris.llms[body.preferred_model] + model = settings.pyris.llms[preferred_model] except ValueError as e: raise InvalidModelException(str(e)) guidance = GuidanceWrapper( model=model, - handlebars=body.template.content, - parameters=body.parameters, + handlebars=template, + parameters=parameters, ) try: return CircuitBreaker.protected_call( func=guidance.query, - cache_key=body.preferred_model, + cache_key=preferred_model, accepted_exceptions=( KeyError, SyntaxError, @@ -60,7 +61,7 @@ def execute_call(body: SendMessageRequest) -> dict: "/api/v1/messages", dependencies=[Depends(TokenPermissionsValidator())] ) def send_message(body: SendMessageRequest) -> SendMessageResponse: - generated_vars = execute_call(body) + generated_vars = execute_call(body.template.content, body.preferred_model, body.parameters) # V1: Throw an exception if no 'response' variable was generated if "response" not in generated_vars: @@ -87,8 +88,8 @@ def send_message(body: SendMessageRequest) -> SendMessageResponse: @router.post( "/api/v2/messages", dependencies=[Depends(TokenPermissionsValidator())] ) -def send_message_v2(body: SendMessageRequest) -> SendMessageResponseV2: - generated_vars = execute_call(body) +def send_message_v2(body: SendMessageRequestV2) -> SendMessageResponseV2: + generated_vars = execute_call(body.template, body.preferred_model, body.parameters) return SendMessageResponseV2( usedModel=body.preferred_model, From 4a9c70d8db31558f22c4177c1f167e7dfb1fbd13 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Thu, 19 Oct 2023 13:13:46 +0200 Subject: [PATCH 14/15] Add V2 message test with different request --- tests/routes/messages_test.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/routes/messages_test.py b/tests/routes/messages_test.py index 3d5ba218..418f82f9 100644 --- a/tests/routes/messages_test.py +++ b/tests/routes/messages_test.py @@ -62,6 +62,31 @@ def test_send_message(test_client, headers, mocker): }, } + +@freeze_time("2023-06-16 03:21:34 +02:00") +@pytest.mark.usefixtures("model_configs") +def test_send_message_v2(test_client, headers, mocker): + mocker.patch.object( + GuidanceWrapper, + "query", + return_value={ + "response": "some content", + }, + autospec=True, + ) + + body = { + "template": "{{#user~}}I want a response to the following query:\ + {{query}}{{~/user}}{{#assistant~}}\ + {{gen 'response' temperature=0.0 max_tokens=500}}{{~/assistant}}", + "preferredModel": "GPT35_TURBO", + "parameters": { + "course": "Intro to Java", + "exercise": "Fun With Sets", + "query": "Some query", + }, + } + response_v2 = test_client.post( "/api/v2/messages", headers=headers, json=body ) From 3b6c9325419a812ed839be7b2ac8ca8f07ce07b2 Mon Sep 17 00:00:00 2001 From: Michael Dyer Date: Thu, 19 Oct 2023 13:15:24 +0200 Subject: [PATCH 15/15] Format --- app/routes/messages.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/routes/messages.py b/app/routes/messages.py index a324ed40..49f96f62 100644 --- a/app/routes/messages.py +++ b/app/routes/messages.py @@ -61,7 +61,9 @@ def execute_call(template, preferred_model, parameters) -> dict: "/api/v1/messages", dependencies=[Depends(TokenPermissionsValidator())] ) def send_message(body: SendMessageRequest) -> SendMessageResponse: - generated_vars = execute_call(body.template.content, body.preferred_model, body.parameters) + generated_vars = execute_call( + body.template.content, body.preferred_model, body.parameters + ) # V1: Throw an exception if no 'response' variable was generated if "response" not in generated_vars: @@ -89,7 +91,9 @@ def send_message(body: SendMessageRequest) -> SendMessageResponse: "/api/v2/messages", dependencies=[Depends(TokenPermissionsValidator())] ) def send_message_v2(body: SendMessageRequestV2) -> SendMessageResponseV2: - generated_vars = execute_call(body.template, body.preferred_model, body.parameters) + generated_vars = execute_call( + body.template, body.preferred_model, body.parameters + ) return SendMessageResponseV2( usedModel=body.preferred_model,