Feat: Add Haiku 3.5 support

epam · Nov 20, 2024 · 579ac81 · 579ac81
1 parent 67d3d6d
commit 579ac81
Show file tree

Hide file tree

Showing 6 changed files with 33 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -16,6 +16,7 @@ Note that a model supports `/truncate_prompt` endpoint if and only if it support
 |Anthropic|Claude 3.5 Sonnet 2.0|[us.]anthropic.claude-3-5-sonnet-20241022-v2:0|text-to-text, image-to-text|🟡|🟡|✅|
 |Anthropic|Claude 3 Sonnet|[us.\|eu.]anthropic.claude-3-sonnet-20240229-v1:0|text-to-text, image-to-text|🟡|🟡|✅|
 |Anthropic|Claude 3 Haiku|[us.\|eu.]anthropic.claude-3-haiku-20240307-v1:0|text-to-text, image-to-text|🟡|🟡|✅|
+|Anthropic|Claude 3.5 Haiku|[us.]anthropic.claude-3-5-haiku-20241022-v1:0|text-to-text|🟡|🟡|✅|
 |Anthropic|Claude 3 Opus|[us.]anthropic.claude-3-opus-20240229-v1:0|text-to-text, image-to-text|🟡|🟡|✅|
 |Anthropic|Claude 2.1|anthropic.claude-v2:1|text-to-text|✅|✅|✅|
 |Anthropic|Claude 2|anthropic.claude-v2|text-to-text|✅|✅|❌|

diff --git a/aidial_adapter_bedrock/deployments.py b/aidial_adapter_bedrock/deployments.py
@@ -33,6 +33,10 @@ class ChatCompletionDeployment(str, Enum):
     ANTHROPIC_CLAUDE_V3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0"
     ANTHROPIC_CLAUDE_V3_HAIKU_US = "us.anthropic.claude-3-haiku-20240307-v1:0"
     ANTHROPIC_CLAUDE_V3_HAIKU_EU = "eu.anthropic.claude-3-haiku-20240307-v1:0"
+    ANTHROPIC_CLAUDE_V3_5_HAIKU = "anthropic.claude-3-5-haiku-20241022-v1:0"
+    ANTHROPIC_CLAUDE_V3_5_HAIKU_US = (
+        "us.anthropic.claude-3-5-haiku-20241022-v1:0"
+    )
     ANTHROPIC_CLAUDE_V3_OPUS = "anthropic.claude-3-opus-20240229-v1:0"
     ANTHROPIC_CLAUDE_V3_OPUS_US = "us.anthropic.claude-3-opus-20240229-v1:0"
 
@@ -92,6 +96,8 @@ def from_deployment_id(
     ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU,
     ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_US,
     ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_EU,
+    ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU,
+    ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU_US,
     ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_OPUS,
     ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_OPUS_US,
 ]

diff --git a/aidial_adapter_bedrock/llm/model/adapter.py b/aidial_adapter_bedrock/llm/model/adapter.py
@@ -57,6 +57,8 @@ async def get_bedrock_adapter(
             | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_SONNET_V2
             | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_SONNET_V2_US
             | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU
+            | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU
+            | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU_US
             | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_US
             | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_EU
             | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_OPUS

diff --git a/aidial_adapter_bedrock/llm/model/claude/v3/tokenizer.py b/aidial_adapter_bedrock/llm/model/claude/v3/tokenizer.py
@@ -192,6 +192,11 @@ def _tokenize_tool_system_message(
             ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU
             | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_US
             | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_EU
+            # Actually token usage for Haiku 3.5 is unknown
+            # temporary using tha same as for Haiku 3
+            # will be not actual after Anthropic tokenizer integration
+            | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU
+            | ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU_US
         ):
             return 264 if tool_choice == "auto" else 340
         case _:

diff --git a/tests/integration_tests/test_chat_completion.py b/tests/integration_tests/test_chat_completion.py
@@ -101,6 +101,8 @@ def get_id(self):
     ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_SONNET_US: _WEST,
     ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_SONNET_V2: _WEST,
     ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_SONNET_V2_US: _WEST,
+    ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU: _WEST,
+    ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU_US: _WEST,
     ChatCompletionDeployment.META_LLAMA3_8B_INSTRUCT_V1: _WEST,
     ChatCompletionDeployment.META_LLAMA3_70B_INSTRUCT_V1: _WEST,
     ChatCompletionDeployment.META_LLAMA3_1_8B_INSTRUCT_V1: _WEST,
@@ -131,6 +133,8 @@ def supports_tools(deployment: ChatCompletionDeployment) -> bool:
         ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU,
         ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_US,
         ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_EU,
+        ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU,
+        ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU_US,
         ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_OPUS,
         ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_OPUS_US,
         ChatCompletionDeployment.META_LLAMA3_1_70B_INSTRUCT_V1,
@@ -182,6 +186,8 @@ def is_claude3(deployment: ChatCompletionDeployment) -> bool:
         ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU,
         ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_US,
         ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_EU,
+        ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU,
+        ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU_US,
         ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_OPUS,
         ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_OPUS_US,
     ]
@@ -202,11 +208,21 @@ def is_ai21(deployment: ChatCompletionDeployment) -> bool:
 
 
 def is_vision_model(deployment: ChatCompletionDeployment) -> bool:
-    return is_claude3(deployment) or deployment in [
+    allowed_models = [
         ChatCompletionDeployment.META_LLAMA3_2_11B_INSTRUCT_V1,
         ChatCompletionDeployment.META_LLAMA3_2_90B_INSTRUCT_V1,
     ]
 
+    excluded_models = {
+        ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU,
+        ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU_US,
+    }
+
+    is_allowed_model = is_claude3(deployment) or deployment in allowed_models
+    is_excluded_model = deployment in excluded_models
+
+    return is_allowed_model and not is_excluded_model
+
 
 def are_tools_emulated(deployment: ChatCompletionDeployment) -> bool:
     return deployment in [

diff --git a/tests/unit_tests/test_endpoints.py b/tests/unit_tests/test_endpoints.py
@@ -25,6 +25,8 @@
     (ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU, True, True),
     (ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_US, True, True),
     (ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_HAIKU_EU, True, True),
+    (ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU, True, True),
+    (ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_5_HAIKU_US, True, True),
     (ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_OPUS, True, True),
     (ChatCompletionDeployment.ANTHROPIC_CLAUDE_V3_OPUS_US, True, True),
     (ChatCompletionDeployment.STABILITY_STABLE_DIFFUSION_XL, False, True),