.Net: Add PostgresVectorStore Memory connector.

dotnet/SK-dotnet.sln

@@ -402,6 +402,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SemanticKernel.AotTests", "
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connectors.Postgres.UnitTests", "src\Connectors\Connectors.Postgres.UnitTests\Connectors.Postgres.UnitTests.csproj", "{232E1153-6366-4175-A982-D66B30AAD610}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Process.Utilities.UnitTests", "src\Experimental\Process.Utilities.UnitTests\Process.Utilities.UnitTests.csproj", "{DAC54048-A39A-4739-8307-EA5A291F2EA0}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -1195,6 +1197,7 @@ Global
 		{6ECFDF04-2237-4A85-B114-DAA34923E9E6} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263}
 		{39EAB599-742F-417D-AF80-95F90376BB18} = {831DDCA2-7D2C-4C31-80DB-6BDB3E1F7AE0}
 		{232E1153-6366-4175-A982-D66B30AAD610} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
+		{DAC54048-A39A-4739-8307-EA5A291F2EA0} = {0D8C6358-5DAA-4EA6-A924-C268A9A21BC9}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {FBDC56A3-86AD-4323-AA0F-201E59123B83}

dotnet/samples/Concepts/Memory/VectorStore_VectorSearch_MultiStore_Postgres.cs

@@ -6,6 +6,7 @@
 using Microsoft.SemanticKernel;
 using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
 using Microsoft.SemanticKernel.Connectors.Postgres;
+using Npgsql;
 
 namespace Memory;
 
@@ -44,7 +45,7 @@ public async Task ExampleWithDIAsync()
 
         // Initialize the Postgres docker container via the fixtures and register the Postgres VectorStore.
         await PostgresFixture.ManualInitializeAsync();
-        kernelBuilder.AddPostgresVectorStore(ConnectionString);
+        kernelBuilder.Services.AddPostgresVectorStore(ConnectionString);
 
         // Register the test output helper common processor with the DI container.
         kernelBuilder.Services.AddSingleton<ITestOutputHelper>(this.Output);
@@ -73,7 +74,10 @@ public async Task ExampleWithoutDIAsync()
 
         // Initialize the Postgres docker container via the fixtures and construct the Postgres VectorStore.
         await PostgresFixture.ManualInitializeAsync();
-        var vectorStore = new PostgresVectorStore(ConnectionString);
+        var dataSourceBuilder = new NpgsqlDataSourceBuilder(ConnectionString);
+        dataSourceBuilder.UseVector();
+        await using var dataSource = dataSourceBuilder.Build();
+        var vectorStore = new PostgresVectorStore(dataSource);
 
         // Create the common processor that works for any vector store.
         var processor = new VectorStore_VectorSearch_MultiStore_Common(vectorStore, textEmbeddingGenerationService, this.Output);

dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreCollectionSqlBuilder.cs

@@ -43,9 +43,11 @@ internal interface IPostgresVectorStoreCollectionSqlBuilder
     /// </summary>
     /// <param name="schema">The schema of the table.</param>
     /// <param name="tableName">The name of the table.</param>
-    /// <param name="vectorProperty">The vector property to create an index for.</param>
+    /// <param name="vectorColumnName">The name of the vector column.</param>
+    /// <param name="indexKind">The kind of index to create.</param>
+    /// <param name="distanceFunction">The distance function to use for the index.</param>
     /// <returns>The built SQL command info.</returns>
-    PostgresSqlCommandInfo BuildCreateVectorIndexCommand(string schema, string tableName, VectorStoreRecordVectorProperty vectorProperty);
+    PostgresSqlCommandInfo BuildCreateVectorIndexCommand(string schema, string tableName, string vectorColumnName, string indexKind, string distanceFunction);
 
     /// <summary>
     /// Builds a SQL command to drop a table in the Postgres vector store.
@@ -127,8 +129,8 @@ internal interface IPostgresVectorStoreCollectionSqlBuilder
     /// <param name="vectorValue">The vector to match.</param>
     /// <param name="filter">The filter conditions for the query.</param>
     /// <param name="skip">The number of records to skip.</param>
-    /// <param name="withEmbeddings">Specifies whether to include embeddings in the result.</param>
+    /// <param name="includeVectors">Specifies whether to include vectors in the result.</param>
     /// <param name="limit">The maximum number of records to return.</param>
     /// <returns>The built SQL command info.</returns>
-    PostgresSqlCommandInfo BuildGetNearestMatchCommand(string schema, string tableName, IReadOnlyList<VectorStoreRecordProperty> properties, VectorStoreRecordVectorProperty vectorProperty, Vector vectorValue, VectorSearchFilter? filter, int? skip, bool withEmbeddings, int limit);
+    PostgresSqlCommandInfo BuildGetNearestMatchCommand(string schema, string tableName, IReadOnlyList<VectorStoreRecordProperty> properties, VectorStoreRecordVectorProperty vectorProperty, Vector vectorValue, VectorSearchFilter? filter, int? skip, bool includeVectors, int limit);
 }

dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreDbClient.cs

@@ -4,15 +4,21 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.VectorData;
+using Npgsql;
 using Pgvector;
 
 namespace Microsoft.SemanticKernel.Connectors.Postgres;
 
 /// <summary>
 /// Internal interface for client managing postgres database operations.
 /// </summary>
-public interface IPostgresVectorStoreDbClient
+internal interface IPostgresVectorStoreDbClient
 {
+    /// <summary>
+    /// The <see cref="NpgsqlDataSource"/> used to connect to the database.
+    /// </summary>
+    public NpgsqlDataSource DataSource { get; }
+
     /// <summary>
     /// Check if a table exists.
     /// </summary>
@@ -28,22 +34,14 @@ public interface IPostgresVectorStoreDbClient
     /// <returns>A group of tables.</returns>
     IAsyncEnumerable<string> GetTablesAsync(CancellationToken cancellationToken = default);
     /// <summary>
-    /// Create a table.
+    /// Create a table. Also creates an index on vector columns if the table has vector properties defined.
     /// </summary>
     /// <param name="tableName">The name assigned to a table of entries.</param>
-    /// <param name="recordDefinition">The record definition of the table.</param>
+    /// <param name="properties">The properties of the record definition that define the table.</param>
     /// <param name="ifNotExists">Specifies whether to include IF NOT EXISTS in the command.</param>
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
     /// <returns></returns>
-    Task CreateTableAsync(string tableName, VectorStoreRecordDefinition recordDefinition, bool ifNotExists = true, CancellationToken cancellationToken = default);
-
-    /// <summary>
-    /// Create a vector index.
-    /// </summary>
-    /// <param name="tableName">The name assigned to a table of entries.</param>
-    /// <param name="vectorProperty">The vector property to create an index for.</param>
-    /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
-    Task CreateVectorIndexAsync(string tableName, VectorStoreRecordVectorProperty vectorProperty, CancellationToken cancellationToken = default);
+    Task CreateTableAsync(string tableName, IReadOnlyList<VectorStoreRecordProperty> properties, bool ifNotExists = true, CancellationToken cancellationToken = default);
 
     /// <summary>
     /// Drop a table.

dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreRecordCollectionFactory.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using Microsoft.Extensions.VectorData;
+using Npgsql;
 
 namespace Microsoft.SemanticKernel.Connectors.Postgres;
 
@@ -14,10 +15,10 @@ public interface IPostgresVectorStoreRecordCollectionFactory
     /// </summary>
     /// <typeparam name="TKey">The data type of the record key.</typeparam>
     /// <typeparam name="TRecord">The data model to use for adding, updating and retrieving data from storage.</typeparam>
-    /// <param name="client">The Postgres client.</param>
+    /// <param name="dataSource">The Postgres data source.</param>
     /// <param name="name">The name of the collection to connect to.</param>
     /// <param name="vectorStoreRecordDefinition">An optional record definition that defines the schema of the record type. If not present, attributes on <typeparamref name="TRecord"/> will be used.</param>
     /// <returns>The new instance of <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/>.</returns>
-    IVectorStoreRecordCollection<TKey, TRecord> CreateVectorStoreRecordCollection<TKey, TRecord>(IPostgresVectorStoreDbClient client, string name, VectorStoreRecordDefinition? vectorStoreRecordDefinition)
+    IVectorStoreRecordCollection<TKey, TRecord> CreateVectorStoreRecordCollection<TKey, TRecord>(NpgsqlDataSource dataSource, string name, VectorStoreRecordDefinition? vectorStoreRecordDefinition)
         where TKey : notnull;
 }

dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresConstants.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Collections.Generic;
+using Microsoft.Extensions.VectorData;
 
 namespace Microsoft.SemanticKernel.Connectors.Postgres;
 
@@ -35,20 +36,29 @@ internal static class PostgresConstants
         typeof(decimal),
         typeof(decimal?),
         typeof(string),
+        typeof(DateTime),
+        typeof(DateTime?),
         typeof(DateTimeOffset),
         typeof(DateTimeOffset?),
         typeof(Guid),
         typeof(Guid?),
         typeof(byte[]),
-        typeof(List<bool>),
-        typeof(List<short>),
-        typeof(List<int>),
-        typeof(List<long>),
-        typeof(List<float>),
-        typeof(List<double>),
-        typeof(List<decimal>),
-        typeof(List<string>),
-        typeof(List<DateTimeOffset>),
+    ];
+
+    /// <summary>A <see cref="HashSet{T}"/> of types that enumerable data properties on the provided model may use as their element types.</summary>
+    public static readonly HashSet<Type> SupportedEnumerableDataElementTypes =
+    [
+        typeof(bool),
+        typeof(short),
+        typeof(int),
+        typeof(long),
+        typeof(float),
+        typeof(double),
+        typeof(decimal),
+        typeof(string),
+        typeof(DateTime),
+        typeof(DateTimeOffset),
+        typeof(Guid),
     ];
 
     /// <summary>A <see cref="HashSet{T}"/> of types that vector properties on the provided model may have.</summary>
@@ -64,4 +74,15 @@ internal static class PostgresConstants
     /// <summary>The name of the column that returns distance value in the database.</summary>
     /// <remarks>It is used in the similarity search query. Must not conflict with model property.</remarks>
     public const string DistanceColumnName = "sk_pg_distance";
+
+    /// <summary>The default index kind.</summary>
+    public const string DefaultIndexKind = IndexKind.Hnsw;
+
+    /// <summary>The default distance function.</summary>
+    public const string DefaultDistanceFunction = DistanceFunction.CosineDistance;
+
+    public static readonly Dictionary<string, int> IndexMaxDimensions = new()
+    {
+        { IndexKind.Hnsw, 2000 },
+    };
 }

dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresGenericDataModelMapper.cs

@@ -14,15 +14,15 @@ internal sealed class PostgresGenericDataModelMapper<TKey> : IVectorStoreRecordM
     /// <summary>
     /// Initializes a new instance of the <see cref="PostgresGenericDataModelMapper{TKey}"/> class.
     /// /// </summary>
-    /// <param name="propertyReader">A <see cref="VectorStoreRecordDefinition"/> that defines the schema of the data in the database.</param>
+    /// <param name="propertyReader"><see cref="VectorStoreRecordPropertyReader"/> with helpers for reading vector store model properties and their attributes.</param>
     public PostgresGenericDataModelMapper(VectorStoreRecordPropertyReader propertyReader)
     {
         Verify.NotNull(propertyReader);
 
         this._propertyReader = propertyReader;
 
         // Validate property types.
-        this._propertyReader.VerifyDataProperties(PostgresConstants.SupportedDataTypes, supportEnumerable: false);
+        this._propertyReader.VerifyDataProperties(PostgresConstants.SupportedDataTypes, PostgresConstants.SupportedEnumerableDataElementTypes);
         this._propertyReader.VerifyVectorProperties(PostgresConstants.SupportedVectorTypes);
     }
     public Dictionary<string, object?> MapFromDataToStorageModel(VectorStoreGenericDataModel<TKey> dataModel)