genai[feat]: add filetype detection for images from bytes (#616)

libs/genai/langchain_google_genai/_image_utils.py

@@ -8,6 +8,7 @@
 from typing import Any, Dict
 from urllib.parse import urlparse
 
+import filetype  # type: ignore[import]
 import requests
 from google.ai.generativelanguage_v1beta.types import Part
 
@@ -87,7 +88,13 @@ def load_part(self, image_string: str) -> Part:
             raise ValueError(msg)
 
         inline_data: Dict[str, Any] = {"data": bytes_}
+
         mime_type, _ = mimetypes.guess_type(image_string)
+        if not mime_type:
+            kind = filetype.guess(bytes_)
+            if kind:
+                mime_type = kind.mime
+
         if mime_type:
             inline_data["mime_type"] = mime_type
 

libs/genai/pyproject.toml

@@ -15,6 +15,7 @@ python = ">=3.9,<4.0"
 langchain-core = ">=0.3.15,<0.4"
 google-generativeai = "^0.8.0"
 pydantic = ">=2,<3"
+filetype = "^1.2.0"
 
 [tool.poetry.group.test]
 optional = true

libs/genai/tests/integration_tests/test_chat_models.py

@@ -158,6 +158,33 @@ def test_chat_google_genai_invoke_multimodal() -> None:
         assert len(chunk.content.strip()) > 0
 
 
+def test_chat_google_genai_invoke_multimodal_by_url() -> None:
+    messages: list = [
+        HumanMessage(
+            content=[
+                {
+                    "type": "text",
+                    "text": "Guess what's in this picture! You have 3 guesses.",
+                },
+                {
+                    "type": "image_url",
+                    "image_url": "https://picsum.photos/seed/picsum/200/300",
+                },
+            ]
+        ),
+    ]
+    llm = ChatGoogleGenerativeAI(model=_VISION_MODEL)
+    response = llm.invoke(messages)
+    assert isinstance(response.content, str)
+    assert len(response.content.strip()) > 0
+
+    # Try streaming
+    for chunk in llm.stream(messages):
+        print(chunk)  # noqa: T201
+        assert isinstance(chunk.content, str)
+        assert len(chunk.content.strip()) > 0
+
+
 def test_chat_google_genai_invoke_multimodal_multiple_messages() -> None:
     messages: list = [
         HumanMessage(content="Hi there"),