.Net: Add PostgresVectorStore Memory connector.

dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresDbClient.cs

@@ -11,6 +11,11 @@ namespace Microsoft.SemanticKernel.Connectors.Postgres;
 /// <summary>
 /// Interface for client managing postgres database operations.
 /// </summary>
+/// <remarks>
+/// This interface is used with the PostgresMemoryStore, which is being deprecated.
+/// Use the <see cref="IPostgresVectorStoreDbClient"/> interface with the PostgresVectorStore
+/// and related classes instead.
+/// </remarks>
 public interface IPostgresDbClient
 {
     /// <summary>

dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreCollectionSqlBuilder.cs

@@ -0,0 +1,125 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using Microsoft.Extensions.VectorData;
+using Pgvector;
+
+namespace Microsoft.SemanticKernel.Connectors.Postgres;
+
+/// <summary>
+/// Interface for constructing SQL commands for Postgres vector store collections.
+/// </summary>
+public interface IPostgresVectorStoreCollectionSqlBuilder
+{
+    /// <summary>
+    /// Builds a SQL command to check if a table exists in the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the table.</param>
+    /// <param name="tableName">The name of the table.</param>
+    /// <returns>The built SQL command.</returns>
+    /// <remarks>
+    /// The command must return a single row with a single column named "table_name" if the table exists.
+    /// </remarks>
+    PostgresSqlCommandInfo BuildDoesTableExistCommand(string schema, string tableName);
+
+    /// <summary>
+    /// Builds a SQL command to fetch all tables in the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the tables.</param>
+    PostgresSqlCommandInfo BuildGetTablesCommand(string schema);
+
+    /// <summary>
+    /// Builds a SQL command to create a table in the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the table.</param>
+    /// <param name="tableName">The name of the table.</param>
+    /// <param name="properties">The properties of the table.</param>
+    /// <param name="ifNotExists">Specifies whether to include IF NOT EXISTS in the command.</param>
+    /// <returns>The built SQL command info.</returns>
+    PostgresSqlCommandInfo BuildCreateTableCommand(string schema, string tableName, IReadOnlyList<VectorStoreRecordProperty> properties, bool ifNotExists = true);
+
+    /// <summary>
+    /// Builds a SQL command to drop a table in the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the table.</param>
+    /// <param name="tableName">The name of the table.</param>
+    /// <returns>The built SQL command info.</returns>
+    PostgresSqlCommandInfo BuildDropTableCommand(string schema, string tableName);
+
+    /// <summary>
+    /// Builds a SQL command to upsert a record in the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the table.</param>
+    /// <param name="tableName">The name of the table.</param>
+    /// <param name="keyColumn">The key column of the table.</param>
+    /// <param name="row">The row to upsert.</param>
+    /// <returns>The built SQL command info.</returns>
+    PostgresSqlCommandInfo BuildUpsertCommand(string schema, string tableName, string keyColumn, Dictionary<string, object?> row);
+
+    /// <summary>
+    /// Builds a SQL command to upsert a batch of records in the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the table.</param>
+    /// <param name="tableName">The name of the table.</param>
+    /// <param name="keyColumn">The key column of the table.</param>
+    /// <param name="rows">The rows to upsert.</param>
+    /// <returns>The built SQL command info.</returns>
+    PostgresSqlCommandInfo BuildUpsertBatchCommand(string schema, string tableName, string keyColumn, List<Dictionary<string, object?>> rows);
+
+    /// <summary>
+    /// Builds a SQL command to get a record from the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the table.</param>
+    /// <param name="tableName">The name of the table.</param>
+    /// <param name="properties">The properties of the table.</param>
+    /// <param name="key">The key of the record to get.</param>
+    /// <param name="includeVectors">Specifies whether to include vectors in the record.</param>
+    /// <returns>The built SQL command info.</returns>
+    PostgresSqlCommandInfo BuildGetCommand<TKey>(string schema, string tableName, IReadOnlyList<VectorStoreRecordProperty> properties, TKey key, bool includeVectors = false) where TKey : notnull;
+
+    /// <summary>
+    /// Builds a SQL command to get a batch of records from the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the table.</param>
+    /// <param name="tableName">The name of the table.</param>
+    /// <param name="properties">The properties of the table.</param>
+    /// <param name="keys">The keys of the records to get.</param>
+    /// <param name="includeVectors">Specifies whether to include vectors in the records.</param>
+    /// <returns>The built SQL command info.</returns>
+    PostgresSqlCommandInfo BuildGetBatchCommand<TKey>(string schema, string tableName, IReadOnlyList<VectorStoreRecordProperty> properties, List<TKey> keys, bool includeVectors = false) where TKey : notnull;
+
+    /// <summary>
+    /// Builds a SQL command to delete a record from the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the table.</param>
+    /// <param name="tableName">The name of the table.</param>
+    /// <param name="keyColumn">The key column of the table.</param>
+    /// <param name="key">The key of the record to delete.</param>
+    /// <returns>The built SQL command info.</returns>
+    PostgresSqlCommandInfo BuildDeleteCommand<TKey>(string schema, string tableName, string keyColumn, TKey key);
+
+    /// <summary>
+    /// Builds a SQL command to delete a batch of records from the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the table.</param>
+    /// <param name="tableName">The name of the table.</param>
+    /// <param name="keyColumn">The key column of the table.</param>
+    /// <param name="keys">The keys of the records to delete.</param>
+    /// <returns>The built SQL command info.</returns>
+    PostgresSqlCommandInfo BuildDeleteBatchCommand<TKey>(string schema, string tableName, string keyColumn, List<TKey> keys);
+
+    /// <summary>
+    /// Builds a SQL command to get the nearest match from the Postgres vector store.
+    /// </summary>
+    /// <param name="schema">The schema of the table.</param>
+    /// <param name="tableName">The name of the table.</param>
+    /// <param name="properties">The properties of the table.</param>
+    /// <param name="vectorProperty">The property which the vectors to compare are stored in.</param>
+    /// <param name="vectorValue">The vector to match.</param>
+    /// <param name="filter">The filter conditions for the query.</param>
+    /// <param name="skip">The number of records to skip.</param>
+    /// <param name="withEmbeddings">Specifies whether to include embeddings in the result.</param>
+    /// <param name="limit">The maximum number of records to return.</param>
+    /// <returns>The built SQL command info.</returns>
+    PostgresSqlCommandInfo BuildGetNearestMatchCommand(string schema, string tableName, IReadOnlyList<VectorStoreRecordProperty> properties, VectorStoreRecordVectorProperty vectorProperty, Vector vectorValue, VectorSearchFilter? filter, int? skip, bool withEmbeddings, int limit);
+}

dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreDbClient.cs

@@ -0,0 +1,164 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.VectorData;
+using Pgvector;
+
+namespace Microsoft.SemanticKernel.Connectors.Postgres;
+
+/// <summary>
+/// Internal interface for client managing postgres database operations.
+/// </summary>
+public interface IPostgresVectorStoreDbClient
+{
+    /// <summary>
+    /// Check if a table exists.
+    /// </summary>
+    /// <param name="tableName">The name assigned to a table of entries.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns></returns>
+    Task<bool> DoesTableExistsAsync(string tableName, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Get all tables.
+    /// </summary>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns>A group of tables.</returns>
+    IAsyncEnumerable<string> GetTablesAsync(CancellationToken cancellationToken = default);
+    /// <summary>
+    /// Create a table.
+    /// </summary>
+    /// <param name="tableName">The name assigned to a table of entries.</param>
+    /// <param name="recordDefinition">The record definition of the table.</param>
+    /// <param name="ifNotExists">Specifies whether to include IF NOT EXISTS in the command.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns></returns>
+    Task CreateTableAsync(string tableName, VectorStoreRecordDefinition recordDefinition, bool ifNotExists = true, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Drop a table.
+    /// </summary>
+    /// <param name="tableName">The name assigned to a table of entries.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    Task DeleteTableAsync(string tableName, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Upsert entry into a table.
+    /// </summary>
+    /// <param name="tableName">The name assigned to a table of entries.</param>
+    /// <param name="row">The row to upsert into the table.</param>
+    /// <param name="keyColumn">The key column of the table.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns></returns>
+    Task UpsertAsync(string tableName, Dictionary<string, object?> row, string keyColumn, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Upsert multiple entries into a table.
+    /// </summary>
+    /// <param name="tableName">The name assigned to a table of entries.</param>
+    /// <param name="rows">The rows to upsert into the table.</param>
+    /// <param name="keyColumn">The key column of the table.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns></returns>
+    Task UpsertBatchAsync(string tableName, IEnumerable<Dictionary<string, object?>> rows, string keyColumn, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Get a entry by its key.
+    /// </summary>
+    /// <param name="tableName">The name assigned to a table of entries.</param>
+    /// <param name="key">The key of the entry to get.</param>
+    /// <param name="properties">The properties to include in the entry.</param>
+    /// <param name="includeVectors">If true, the vectors will be included in the entry.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns>The row if the key is found, otherwise null.</returns>
+    Task<Dictionary<string, object?>?> GetAsync<TKey>(string tableName, TKey key, IReadOnlyList<VectorStoreRecordProperty> properties, bool includeVectors = false, CancellationToken cancellationToken = default)
+        where TKey : notnull;
+
+    /// <summary>
+    /// Get multiple entries by their keys.
+    /// </summary>
+    /// <param name="tableName">The name assigned to a table of entries.</param>
+    /// <param name="keys">The keys of the entries to get.</param>
+    /// <param name="properties">The properties of the table.</param>
+    /// <param name="includeVectors">If true, the vectors will be included in the entries.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns>The rows that match the given keys.</returns>
+    IAsyncEnumerable<Dictionary<string, object?>> GetBatchAsync<TKey>(string tableName, IEnumerable<TKey> keys, IReadOnlyList<VectorStoreRecordProperty> properties, bool includeVectors = false, CancellationToken cancellationToken = default)
+        where TKey : notnull;
+
+    /// <summary>
+    /// Delete a entry by its key.
+    /// </summary>
+    /// <param name="tableName">The name assigned to a table of entries.</param>
+    /// <param name="keyColumn">The name of the key column.</param>
+    /// <param name="key">The key of the entry to delete.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns></returns>
+    Task DeleteAsync<TKey>(string tableName, string keyColumn, TKey key, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Delete multiple entries by their keys.
+    /// </summary>
+    /// <param name="tableName">The name assigned to a table of entries.</param>
+    /// <param name="keyColumn">The name of the key column.</param>
+    /// <param name="keys">The keys of the entries to delete.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns></returns>
+    Task DeleteBatchAsync<TKey>(string tableName, string keyColumn, IEnumerable<TKey> keys, CancellationToken cancellationToken = default);
+
+    /// <summary>
+    /// Gets the nearest matches to the <see cref="Vector"/>.
+    /// </summary>
+    /// <param name="tableName">The name assigned to a table of entries.</param>
+    /// <param name="properties">The properties to retrieve.</param>
+    /// <param name="vectorProperty">The property which the vectors to compare are stored in.</param>
+    /// <param name="vectorValue">The <see cref="Vector"/> to compare the table's vector with.</param>
+    /// <param name="limit">The maximum number of similarity results to return.</param>
+    /// <param name="filter">Optional conditions to filter the results.</param>
+    /// <param name="skip">The number of entries to skip.</param>
+    /// <param name="includeVectors">If true, the vectors will be returned in the entries.</param>
+    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    /// <returns>An asynchronous stream of <see cref="PostgresMemoryEntry"/> objects that the nearest matches to the <see cref="Vector"/>.</returns>
+    IAsyncEnumerable<(Dictionary<string, object?> Row, double Distance)> GetNearestMatchesAsync(string tableName, IReadOnlyList<VectorStoreRecordProperty> properties, VectorStoreRecordVectorProperty vectorProperty, Vector vectorValue, int limit,
+        VectorSearchFilter? filter = default, int? skip = default, bool includeVectors = false, CancellationToken cancellationToken = default);
+
+    // /// <summary>
+    // /// Read a entry by its key.
+    // /// </summary>
+    // /// <param name="tableName">The name assigned to a table of entries.</param>
+    // /// <param name="key">The key of the entry to read.</param>
+    // /// <param name="withEmbeddings">If true, the embeddings will be returned in the entry.</param>
+    // /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    // /// <returns></returns>
+    // Task<PostgresMemoryEntry?> ReadAsync(string tableName, string key, bool withEmbeddings = false, CancellationToken cancellationToken = default);
+
+    // /// <summary>
+    // /// Read multiple entries by their keys.
+    // /// </summary>
+    // /// <param name="tableName">The name assigned to a table of entries.</param>
+    // /// <param name="keys">The keys of the entries to read.</param>
+    // /// <param name="withEmbeddings">If true, the embeddings will be returned in the entries.</param>
+    // /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    // /// <returns>An asynchronous stream of <see cref="PostgresMemoryEntry"/> objects that match the given keys.</returns>
+    // IAsyncEnumerable<PostgresMemoryEntry> ReadBatchAsync(string tableName, IEnumerable<string> keys, bool withEmbeddings = false, CancellationToken cancellationToken = default);
+
+    // /// <summary>
+    // /// Delete a entry by its key.
+    // /// </summary>
+    // /// <param name="tableName">The name assigned to a table of entries.</param>
+    // /// <param name="key">The key of the entry to delete.</param>
+    // /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    // /// <returns></returns>
+    // Task DeleteAsync(string tableName, string key, CancellationToken cancellationToken = default);
+
+    // /// <summary>
+    // /// Delete multiple entries by their key.
+    // /// </summary>
+    // /// <param name="tableName">The name assigned to a table of entries.</param>
+    // /// <param name="keys">The keys of the entries to delete.</param>
+    // /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
+    // /// <returns></returns>
+    // Task DeleteBatchAsync(string tableName, IEnumerable<string> keys, CancellationToken cancellationToken = default);
+}

dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreRecordCollectionFactory.cs

@@ -0,0 +1,23 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.Extensions.VectorData;
+
+namespace Microsoft.SemanticKernel.Connectors.Postgres;
+
+/// <summary>
+/// Interface for constructing <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/> Postgres instances when using <see cref="IVectorStore"/> to retrieve these.
+/// </summary>
+public interface IPostgresVectorStoreRecordCollectionFactory
+{
+    /// <summary>
+    /// Constructs a new instance of the <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/>.
+    /// </summary>
+    /// <typeparam name="TKey">The data type of the record key.</typeparam>
+    /// <typeparam name="TRecord">The data model to use for adding, updating and retrieving data from storage.</typeparam>
+    /// <param name="client">The Postgres client.</param>
+    /// <param name="name">The name of the collection to connect to.</param>
+    /// <param name="vectorStoreRecordDefinition">An optional record definition that defines the schema of the record type. If not present, attributes on <typeparamref name="TRecord"/> will be used.</param>
+    /// <returns>The new instance of <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/>.</returns>
+    IVectorStoreRecordCollection<TKey, TRecord> CreateVectorStoreRecordCollection<TKey, TRecord>(IPostgresVectorStoreDbClient client, string name, VectorStoreRecordDefinition? vectorStoreRecordDefinition)
+        where TKey : notnull;
+}