feat: supported llama 3.1 models (#136)

epam · Aug 12, 2024 · 9110dd8 · 9110dd8
1 parent f5d734f
commit 9110dd8
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -17,6 +17,9 @@ The following models support `POST SERVER_URL/openai/deployments/DEPLOYMENT_NAME
 |Anthropic|Claude 2.1|anthropic.claude-v2:1|text-to-text|✅|✅|✅|✅|
 |Anthropic|Claude 2|anthropic.claude-v2|text-to-text|✅|✅|❌|✅|
 |Anthropic|Claude Instant 1.2|anthropic.claude-instant-v1|text-to-text|✅|✅|❌|❌|
+|Meta|Llama 3.1 405B Instruct|meta.llama3-1-405b-instruct-v1:0|text-to-text|✅|✅|❌|❌|
+|Meta|Llama 3.1 70B Instruct|meta.llama3-1-70b-instruct-v1:0|text-to-text|✅|✅|❌|❌|
+|Meta|Llama 3.1 8B Instruct|meta.llama3-1-8b-instruct-v1:0|text-to-text|✅|✅|❌|❌|
 |Meta|Llama 3 Chat 70B Instruct|meta.llama3-70b-instruct-v1:0|text-to-text|✅|✅|❌|❌|
 |Meta|Llama 3 Chat 8B Instruct|meta.llama3-8b-instruct-v1:0|text-to-text|✅|✅|❌|❌|
 |Meta|Llama 2 Chat 70B|meta.llama2-70b-chat-v1|text-to-text|✅|✅|❌|❌|

diff --git a/aidial_adapter_bedrock/deployments.py b/aidial_adapter_bedrock/deployments.py
@@ -3,21 +3,29 @@
 
 class ChatCompletionDeployment(str, Enum):
     AMAZON_TITAN_TG1_LARGE = "amazon.titan-tg1-large"
+
     AI21_J2_GRANDE_INSTRUCT = "ai21.j2-grande-instruct"
     AI21_J2_JUMBO_INSTRUCT = "ai21.j2-jumbo-instruct"
+
     ANTHROPIC_CLAUDE_INSTANT_V1 = "anthropic.claude-instant-v1"
     ANTHROPIC_CLAUDE_V2 = "anthropic.claude-v2"
     ANTHROPIC_CLAUDE_V2_1 = "anthropic.claude-v2:1"
     ANTHROPIC_CLAUDE_V3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0"
     ANTHROPIC_CLAUDE_V3_5_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0"
     ANTHROPIC_CLAUDE_V3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0"
     ANTHROPIC_CLAUDE_V3_OPUS = "anthropic.claude-3-opus-20240229-v1:0"
+
     STABILITY_STABLE_DIFFUSION_XL = "stability.stable-diffusion-xl"
     STABILITY_STABLE_DIFFUSION_XL_V1 = "stability.stable-diffusion-xl-v1"
+
     META_LLAMA2_13B_CHAT_V1 = "meta.llama2-13b-chat-v1"
     META_LLAMA2_70B_CHAT_V1 = "meta.llama2-70b-chat-v1"
     META_LLAMA3_8B_INSTRUCT_V1 = "meta.llama3-8b-instruct-v1:0"
     META_LLAMA3_70B_INSTRUCT_V1 = "meta.llama3-70b-instruct-v1:0"
+    META_LLAMA3_1_405B_INSTRUCT_V1 = "meta.llama3-1-405b-instruct-v1:0"
+    META_LLAMA3_1_70B_INSTRUCT_V1 = "meta.llama3-1-70b-instruct-v1:0"
+    META_LLAMA3_1_8B_INSTRUCT_V1 = "meta.llama3-1-8b-instruct-v1:0"
+
     COHERE_COMMAND_TEXT_V14 = "cohere.command-text-v14"
     COHERE_COMMAND_LIGHT_TEXT_V14 = "cohere.command-light-text-v14"
 

diff --git a/aidial_adapter_bedrock/llm/model/adapter.py b/aidial_adapter_bedrock/llm/model/adapter.py
@@ -77,6 +77,9 @@ async def get_bedrock_adapter(
         case (
             ChatCompletionDeployment.META_LLAMA3_8B_INSTRUCT_V1
             | ChatCompletionDeployment.META_LLAMA3_70B_INSTRUCT_V1
+            | ChatCompletionDeployment.META_LLAMA3_1_405B_INSTRUCT_V1
+            | ChatCompletionDeployment.META_LLAMA3_1_70B_INSTRUCT_V1
+            | ChatCompletionDeployment.META_LLAMA3_1_8B_INSTRUCT_V1
         ):
             return MetaAdapter.create(
                 await Bedrock.acreate(region), model, llama3_config

diff --git a/tests/integration_tests/test_chat_completion.py b/tests/integration_tests/test_chat_completion.py
@@ -118,6 +118,9 @@ def get_id(self):
     ChatCompletionDeployment.META_LLAMA2_70B_CHAT_V1,
     ChatCompletionDeployment.META_LLAMA3_8B_INSTRUCT_V1,
     ChatCompletionDeployment.META_LLAMA3_70B_INSTRUCT_V1,
+    ChatCompletionDeployment.META_LLAMA3_1_405B_INSTRUCT_V1,
+    ChatCompletionDeployment.META_LLAMA3_1_70B_INSTRUCT_V1,
+    ChatCompletionDeployment.META_LLAMA3_1_8B_INSTRUCT_V1,
     ChatCompletionDeployment.COHERE_COMMAND_TEXT_V14,
 ]
 
@@ -267,7 +270,7 @@ def test_case(
     )
 
     test_case(
-        name="max tokens 1",
+        name="pinocchio in one token",
         max_tokens=1,
         messages=[user("tell me the full story of Pinocchio")],
         expected=lambda s: len(s.content.split()) <= 1,

diff --git a/tests/unit_tests/test_endpoints.py b/tests/unit_tests/test_endpoints.py
@@ -23,6 +23,9 @@
     (ChatCompletionDeployment.META_LLAMA2_70B_CHAT_V1, True, True),
     (ChatCompletionDeployment.META_LLAMA3_8B_INSTRUCT_V1, True, True),
     (ChatCompletionDeployment.META_LLAMA3_70B_INSTRUCT_V1, True, True),
+    (ChatCompletionDeployment.META_LLAMA3_1_405B_INSTRUCT_V1, True, True),
+    (ChatCompletionDeployment.META_LLAMA3_1_70B_INSTRUCT_V1, True, True),
+    (ChatCompletionDeployment.META_LLAMA3_1_8B_INSTRUCT_V1, True, True),
     (ChatCompletionDeployment.COHERE_COMMAND_TEXT_V14, True, True),
     (ChatCompletionDeployment.COHERE_COMMAND_LIGHT_TEXT_V14, True, True),
 ]