Image prompt template

[hinthornw]

Builds on Bagatur's (#13227). See unit test for example usage (below)

def test_chat_tmpl_from_messages_multipart_image() -> None:
    base64_image = "abcd123"
    other_base64_image = "abcd123"
    template = ChatPromptTemplate.from_messages(
        [
            ("system", "You are an AI assistant named {name}."),
            (
                "human",
                [
                    {"type": "text", "text": "What's in this image?"},
                    # OAI supports all these structures today
                    {
                        "type": "image_url",
                        "image_url": "data:image/jpeg;base64,{my_image}",
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": "data:image/jpeg;base64,{my_image}"},
                    },
                    {"type": "image_url", "image_url": "{my_other_image}"},
                    {
                        "type": "image_url",
                        "image_url": {"url": "{my_other_image}", "detail": "medium"},
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": "https://www.langchain.com/image.png"},
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": "data:image/jpeg;base64,foobar"},
                    },
                ],
            ),
        ]
    )
    messages = template.format_messages(
        name="R2D2", my_image=base64_image, my_other_image=other_base64_image
    )
    expected = [
        SystemMessage(content="You are an AI assistant named R2D2."),
        HumanMessage(
            content=[
                {"type": "text", "text": "What's in this image?"},
                {
                    "type": "image_url",
                    "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"},
                },
                {
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:image/jpeg;base64,{other_base64_image}"
                    },
                },
                {
                    "type": "image_url",
                    "image_url": {"url": f"{other_base64_image}"},
                },
                {
                    "type": "image_url",
                    "image_url": {
                        "url": f"{other_base64_image}",
                        "detail": "medium",
                    },
                },
                {
                    "type": "image_url",
                    "image_url": {"url": "https://www.langchain.com/image.png"},
                },
                {
                    "type": "image_url",
                    "image_url": {"url": "data:image/jpeg;base64,foobar"},
                },
            ]
        ),
    ]
    assert messages == expected

[vercel]

The latest updates on your projects. Learn more about Vercel for Git ↗︎

1 Ignored Deployment

Name	Status	Preview	Comments	Updated (UTC)
langchain	⬜️ Ignored (Inspect)	Visit Preview		Jan 28, 2024 1:00am

[rlancemartin]

Tested -
#14263
langchain-ai/langchain-benchmarks#106

[hinthornw]

We could also make these accessible as an instance of ImagePromptValue.to_b64_string etc.

[hinthornw]

These actually aren't required to be blocked out now that I'm using the fstring format to infer again

[baskaryan]

this type of prompt wouldn't work anymore i take it?

prompt = ChatPromptTemplate.from_messages([
    ("system", "foo bar"),
    ("human", [
        "{question}",
        {"text": "{second_question}"},
        # image with var name 'first_image' and no preset values
        {"image_url": "{first_image}"}, 
        # image with var name 'second_image' and preset "detail" level
        {"image_url": {"variable_name": "second_image", "detail": "low"}}, 
        # constant image, not a var
        {"type": "image_url", "image_url": {"url": "https://python.langchain.com/assets/images/langchain_stack-f195a2c417953cb5828f76e5f9039c9f.png"}}]), 
])

[hinthornw]

   {"image_url": {"variable_name": "second_image", "detail": "low"}}, 

^ this one wouldn't work right now. Is that a preferable format?