format prompt messages with openai image content (langchain-ai#3212)

* format prompt messages with openai image content

* cleanup

langchain/src/prompts/chat.ts

@@ -559,17 +559,51 @@ export class ChatPromptTemplate<
     return "chat";
   }
 
+  private async _parseImagePrompts(
+    message: BaseMessage,
+    inputValues: InputValues<
+      PartialVariableName | Extract<keyof RunInput, string>
+    >
+  ): Promise<BaseMessage> {
+    if (typeof message.content === "string") {
+      return message;
+    }
+    const formattedMessageContent = await Promise.all(
+      message.content.map(async (item) => {
+        if (
+          item.type !== "image_url" ||
+          typeof item.image_url === "string" ||
+          !item.image_url?.url
+        ) {
+          return item;
+        }
+        const imageUrl = item.image_url.url;
+        const promptTemplatePlaceholder = PromptTemplate.fromTemplate(imageUrl);
+        const formattedUrl = await promptTemplatePlaceholder.format(
+          inputValues
+        );
+        // eslint-disable-next-line no-param-reassign
+        item.image_url.url = formattedUrl;
+        return item;
+      })
+    );
+    // eslint-disable-next-line no-param-reassign
+    message.content = formattedMessageContent;
+    return message;
+  }
+
   async formatMessages(
     values: TypedPromptInputValues<RunInput>
   ): Promise<BaseMessage[]> {
     const allValues = await this.mergePartialAndUserVariables(values);
-
     let resultMessages: BaseMessage[] = [];
 
     for (const promptMessage of this.promptMessages) {
       // eslint-disable-next-line no-instanceof/no-instanceof
       if (promptMessage instanceof BaseMessage) {
-        resultMessages.push(promptMessage);
+        resultMessages.push(
+          await this._parseImagePrompts(promptMessage, allValues)
+        );
       } else {
         const inputValues = promptMessage.inputVariables.reduce(
           (acc, inputVariable) => {

langchain/src/schema/tests/base_message.test.ts

@@ -0,0 +1,32 @@
+import { ChatPromptTemplate } from "../../prompts/chat.js";
+import { HumanMessage } from "../index.js";
+
+test("Test ChatPromptTemplate can format OpenAI content image messages", async () => {
+  const message = new HumanMessage({
+    content: [
+      {
+        type: "image_url",
+        image_url: {
+          url: `data:image/jpeg;base64,{image_string}`,
+        },
+      },
+    ],
+  });
+  const prompt = ChatPromptTemplate.fromMessages([
+    message,
+    ["ai", "Will this format with multiple messages?: {yes_or_no}"],
+  ]);
+  const formatted = await prompt.invoke({
+    image_string: "base_64_encoded_string",
+    yes_or_no: "YES!",
+  });
+  expect(formatted.messages[0].content[0]).toEqual({
+    type: "image_url",
+    image_url: {
+      url: "data:image/jpeg;base64,base_64_encoded_string",
+    },
+  });
+  expect(formatted.messages[1].content).toEqual(
+    "Will this format with multiple messages?: YES!"
+  );
+});