How to add history and memory functionality to multi modal rag setup

[saichandra1199]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

retriever_engine = index.as_retriever(
    similarity_top_k=5, image_similarity_top_k=5
)

metadata_vid = {'views':1000,'title':'Science',"likes":2000}

import json
from llama_index.core.response.notebook_utils import display_source_node
from llama_index.core.schema import ImageNode
from llama_index.multi_modal_llms.ollama import OllamaMultiModal

metadata_str = json.dumps(metadata_vid)

qa_tmpl_str = (
    "Given the provided information, including relevant images and retrieved context from the video, \
 accurately and precisely answer the query without any additional prior knowledge.\n"
    "Please ensure honesty and responsibility, refraining from any racist or sexist remarks.\n"
    "---------------------\n"
    "Context: {context_str}\n"
    "Metadata for video: {metadata_str} \n"
    "---------------------\n"
    "Query: {query_str}\n"
    "Answer: "
)

ollama_mm_llm = OllamaMultiModal(
    model="llava",  
    max_new_tokens=1500
)

def retrieve(retriever_engine, query_str):
    retrieval_results = retriever_engine.retrieve(query_str)

    retrieved_image = []
    retrieved_text = []
    for res_node in retrieval_results:
        if isinstance(res_node.node, ImageNode):
            retrieved_image.append(res_node.node.metadata["file_path"])
        else:
            display_source_node(res_node, source_length=200)
            retrieved_text.append(res_node.text)

    return retrieved_image, retrieved_text

query_str = " Until the early 20th century whose laws are used to explain physics ?"

img, txt = retrieve(retriever_engine=retriever_engine, query_str=query_str)
image_documents = SimpleDirectoryReader(
    input_dir=output_folder, input_files=img 
    ).load_data()
context_str = "".join(txt)
plot_images(img)


# Use the model to complete a prompt with image documents
response_1 = ollama_mm_llm.complete(
    prompt=qa_tmpl_str.format(
        context_str=context_str, query_str=query_str, metadata_str=metadata_str
    ),
    image_documents=image_documents,
)

print("Question : ",query_str)
print("Answer : ",response_1.text)

Description

I'm trying to add memory and conversation type functionality based on the context and history of the previous queries.

What modifications in the above code is needed to run this code as a chatbot

System Info

langchain==0.3.7
langchain-community==0.3.5
langchain-core==0.3.15
langchain-text-splitters==0.3.2
llama-index-embeddings-langchain==0.2.1