Is it possible to get citation from AI Search in RAG implementaiton using function?

[PanClifton]

Hello,

I am implementing RAG using KernelFunction and AI Search. How can i return information(citation) to the user that agent give an answer based on specific document(in my case blob link) in index. Something similar to citation in kernel memory.

this is my setup

function

using Azure.Search.Documents.Indexes;
using Azure.Search.Documents.Models;
using Azure.Search.Documents;
using Azure;
using Microsoft.SemanticKernel.Embeddings;
using Microsoft.SemanticKernel;
using System.ComponentModel;
using System.Text.Json.Serialization;

namespace HeiDi.Sandbox.SemanticKernelPoc.AiSearch.Plugins
{

#pragma warning disable SKEXP0001
#pragma warning disable SKEXP0010
#pragma warning disable SKEXP0020

    public class AzureAISearchPlugin
    {
        const string CONTRACT_INDEX_NAME = "contracts-index";
        private readonly ITextEmbeddingGenerationService _textEmbeddingGenerationService;
        private readonly SearchIndexClient _indexClient;

        public AzureAISearchPlugin(ITextEmbeddingGenerationService textEmbeddingGenerationService, SearchIndexClient indexClient)
        {
            _textEmbeddingGenerationService = textEmbeddingGenerationService;
            _indexClient = indexClient;
        }

        [KernelFunction("private_data")]
        [Description("Search for a data in documents matching given query in user input.")]
        [return: Description("Collection of documents with content, title and file source")]
        public async Task<IEnumerable<DocmentInfo>> SearchAsync(string userInput)
        {
            ReadOnlyMemory<float> embedding = await _textEmbeddingGenerationService.GenerateEmbeddingAsync(userInput);

            SearchClient searchClient = _indexClient.GetSearchClient(CONTRACT_INDEX_NAME);

            VectorizedQuery vectorQuery = new(embedding);
            vectorQuery.Fields.Add("content_vector");

            SearchOptions searchOptions = new()
            {
                VectorSearch = new()
                {
                    Queries = { vectorQuery }
                }
            };

            Response<SearchResults<IndexSchema>> response = await searchClient.SearchAsync<IndexSchema>(searchOptions);

            var searchResult = new List<DocmentInfo>();
            await foreach (SearchResult<IndexSchema> result in response.Value.GetResultsAsync())
            {
                searchResult.Add(new DocmentInfo(result.Document.Title, result.Document.Content, result.Document.Source));
            }

            return searchResult;
        }

        public record DocmentInfo(string Titile, string Content, string FileSource);

        public sealed class IndexSchema
        {
            [JsonPropertyName("content")]
            public string? Content { get; set; }

            [JsonPropertyName("chunk")]
            public int? Chunk { get; set; }


            [JsonPropertyName("metadata")]
            public string? Metadata { get; set; }


            [JsonPropertyName("title")]
            public string? Title { get; set; }

            [JsonPropertyName("source")]
            public string? Source { get; set; }

            [JsonPropertyName("source_id")]
            public string? SourceId { get; set; }

            [JsonPropertyName("filepath")]
            public string? Filepath { get; set; }

        }
    }
}

program cs

using Azure.Search.Documents.Indexes;
using Azure;
using HeiDi.Sandbox.SemanticKernelPoc.AiSearch.Plugins;
using Microsoft.SemanticKernel.Agents;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Connectors.OpenAI;
using Microsoft.SemanticKernel.Embeddings;
using Microsoft.Extensions.DependencyInjection;
using HeiDi.Sandbox.SemanticKernelPoc.AiSearch.Agents;
using HeiDi.Sandbox.SemanticKernelPoc.Utils.Providers;
using Microsoft.SemanticKernel.ChatCompletion;

#pragma warning disable SKEXP0001
#pragma warning disable SKEXP0010
#pragma warning disable SKEXP0110
#pragma warning disable SKEXP0020

var configurationProvider = ConfigurationsProvider.CreateNew();
var aiSearchConfigrationy = configurationProvider.AiSearchConfiguration;
var openAiConfigration = configurationProvider.OpenAiConfigration;

Uri endpoint = new(aiSearchConfigrationy.Endpoint);
AzureKeyCredential keyCredential = new(aiSearchConfigrationy.ApiKey);

IKernelBuilder kernelBuilder = Kernel.CreateBuilder();

//kernelBuilder.Services.AddLogging(services => services.AddConsole().SetMinimumLevel(LogLevel.Trace));
kernelBuilder.Services.AddSingleton<ITextEmbeddingGenerationService, OpenAITextEmbeddingGenerationService>();

kernelBuilder.Services.AddSingleton<SearchIndexClient>((_) => new SearchIndexClient(endpoint, keyCredential));

kernelBuilder.AddAzureOpenAITextEmbeddingGeneration(openAiConfigration.TextEmbeddedModelDelpoymentName, openAiConfigration.Endpoint, openAiConfigration.ApiKey);

kernelBuilder.AddAzureOpenAIChatCompletion(openAiConfigration.ModelDelpoymentName, openAiConfigration.Endpoint, openAiConfigration.ApiKey);

kernelBuilder.Plugins.AddFromType<AzureAISearchPlugin>();

// Create kernel
var kernel = kernelBuilder.Build();


ChatCompletionAgent agent = new()
{
    Description = Agents.Instance.AiSearchAgent.Description,
    Instructions = Agents.Instance.AiSearchAgent.Instructions,
    Name = "AzureSearchAgent",
    Kernel = kernel,
    Arguments = new KernelArguments(new OpenAIPromptExecutionSettings() { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }),
};

Console.WriteLine("Assistant >  How can I help you today?");

while (true)
{
    Console.Write("User > ");
    var input = Console.ReadLine();
    if (input == "exit")
    {
        break;
    }
    var chat = new AgentGroupChat(agent);


    chat.AddChatMessage(new ChatMessageContent(AuthorRole.User, input));
    await foreach (var content in chat.InvokeAsync(agent))
    {
        Console.WriteLine($"# {content.Role} - {content.AuthorName ?? "*"}: '{content.Content}'");        
    }
}

[dmytrostruk]

Hi @PanClifton , thanks for your question. In your DocumentInfo model, would it be possible to add another property Citation and include a link from IndexSchema.Filepath property? In that case, the model should return an information about requested data together with file path/link. Please let me know if that helps. Thank you!

[PanClifton]

Hi @dmytrostruk thanks for the answer

this solution is no good for me since model dont return IndexSchema.Filepath every time. i was wondering is there is a solution to force model to use Azure AI search all the time and give me back citetions

[PanClifton]

@dmytrostruk i think that i need to abandon rag solution with function in favour for following solution:

#pragma warning disable SKEXP0001
#pragma warning disable SKEXP0010
#pragma warning disable SKEXP0110
#pragma warning disable SKEXP0020
#pragma warning disable AOAI001

using Azure.AI.OpenAI.Chat;
using HeiDi.Sandbox.Utils.Providers;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;
using Microsoft.SemanticKernel.Connectors.AzureOpenAI;

var configurationProvider = ConfigurationsProvider.CreateNew();

var aiSearchConfigration = configurationProvider.AiSearchConfiguration;
var openAiConfigration = configurationProvider.OpenAiConfigration;

var datasource = new AzureSearchChatDataSource
{
    Endpoint = new Uri(aiSearchConfigration.Endpoint),
    Authentication = DataSourceAuthentication.FromApiKey(aiSearchConfigration.ApiKey),
    IndexName = "contracts-index"
};

var kernel = Kernel.CreateBuilder()
      .AddAzureOpenAIChatCompletion(
          openAiConfigration.ModelDelpoymentName,
         openAiConfigration.Endpoint,
         openAiConfigration.ApiKey)
      .Build();
var promptExecutionSettings = new AzureOpenAIPromptExecutionSettings { AzureChatDataSource = datasource };

var chatHistory = new ChatHistory();

var ask = "Could you describe the change request process including all steps?";
chatHistory.AddUserMessage(ask);

var chatCompletion = kernel.GetRequiredService<IChatCompletionService>();

var chatMessage = await chatCompletion.GetChatMessageContentAsync(chatHistory, promptExecutionSettings);

var response = chatMessage.Content!;

var message = chatMessage.InnerContent as OpenAI.Chat.ChatCompletion;
var messageContext = message.GetMessageContext();
//
// citations extracted
//
var citations = messageContext.Citations;

Console.WriteLine(response);

but is it possible to extract citations from StreamingChatMessageContent?

#pragma warning disable SKEXP0001
#pragma warning disable SKEXP0010
#pragma warning disable SKEXP0110
#pragma warning disable SKEXP0020
#pragma warning disable AOAI001

using Azure.AI.OpenAI.Chat;
using HeiDi.Sandbox.Utils.Providers;
using Microsoft.Extensions.AI;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;
using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
using OpenAI.Chat;

var configurationProvider = ConfigurationsProvider.CreateNew();

var aiSearchConfigration = configurationProvider.AiSearchConfiguration;
var openAiConfigration = configurationProvider.OpenAiConfigration;

var datasource = new AzureSearchChatDataSource
{
    Endpoint = new Uri(aiSearchConfigration.Endpoint),
    Authentication = DataSourceAuthentication.FromApiKey(aiSearchConfigration.ApiKey),
    IndexName = "contracts-index"
};

var kernel = Kernel.CreateBuilder()
      .AddAzureOpenAIChatCompletion(
          openAiConfigration.ModelDelpoymentName,
         openAiConfigration.Endpoint,
         openAiConfigration.ApiKey)
      .Build();
var promptExecutionSettings = new AzureOpenAIPromptExecutionSettings { AzureChatDataSource = datasource };

var chatHistory = new ChatHistory();

var ask = "Could you describe the change request process including all steps?";
chatHistory.AddUserMessage(ask);

var chatCompletion = kernel.GetRequiredService<IChatCompletionService>();

await foreach (StreamingChatMessageContent content in chatCompletion.GetStreamingChatMessageContentsAsync(chatHistory, promptExecutionSettings))
{
    //
    // is it possible to extract citations from content?
    //
};

[dmytrostruk]

@PanClifton Yes, it's possible to extract citations from StreamingChatMessageContent:

await foreach (StreamingChatMessageContent content in chatCompletion.GetStreamingChatMessageContentsAsync(chatHistory, promptExecutionSettings))
{
    var update = content.InnerContent as OpenAI.Chat.StreamingChatCompletionUpdate;
    var messageContext = update?.GetMessageContext();

    var citations = messageContext?.Citations;
};

I also updated our example of Azure Chat Completion with data to show how to get citations in streaming mode in this PR: #9760

[PanClifton]

Thanks @dmytrostruk