Skip to content

Commit

Permalink
test: Fixed database tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
@HavenDV
HavenDV committed Aug 6, 2024
1 parent a48d02a commit 1d08711
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 53 deletions.
114 changes: 62 additions & 52 deletions src/Databases/IntegrationTests/DatabaseTests.Configure.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
using Testcontainers.PostgreSql;
using LangChain.Databases.Weaviate;
using Microsoft.SemanticKernel.Connectors.Weaviate;
using Testcontainers.Milvus;

namespace LangChain.Databases.IntegrationTests;

Expand Down Expand Up @@ -151,72 +152,81 @@ private static async Task<DatabaseTestEnvironment> StartEnvironmentForAsync(Supp
}
case SupportedDatabase.Milvus:
{
var network = new NetworkBuilder()
.WithName("milvus-network")
.Build();

var etcdContainer = new ContainerBuilder()
.WithImage("quay.io/coreos/etcd:v3.5.5")
.WithName("milvus-etcd")
.WithEnvironment("ETCD_AUTO_COMPACTION_MODE", "revision")
.WithEnvironment("ETCD_AUTO_COMPACTION_RETENTION", "1000")
.WithEnvironment("ETCD_QUOTA_BACKEND_BYTES", "4294967296")
.WithEnvironment("ETCD_SNAPSHOT_COUNT", "50000")
.WithPortBinding(2379, 2379)
.WithCommand("etcd",
"-advertise-client-urls=http://0.0.0.0:2379",
"-listen-client-urls=http://0.0.0.0:2379",
"--data-dir", "/etcd")
.Build();

var minioContainer = new ContainerBuilder()
.WithImage("minio/minio:RELEASE.2023-03-20T20-16-18Z")
.WithName("milvus-minio")
.WithPortBinding(9000, 9000)
.WithPortBinding(9001, 9001)
.WithEnvironment("MINIO_ACCESS_KEY", "minioadmin")
.WithEnvironment("MINIO_SECRET_KEY", "minioadmin")
.WithCommand("minio", "server", "/minio_data", "--console-address", ":9001")
.Build();

var milvusContainer = new ContainerBuilder()
.WithImage("milvusdb/milvus:v2.3.0")
.WithName("milvus-standalone")
.WithPortBinding(19530, 19530)
.WithPortBinding(9091, 9091)
.WithEnvironment("ETCD_ENDPOINTS", "milvus-etcd:2379")
.WithEnvironment("MINIO_ADDRESS", "milvus-minio:9000")
.WithCommand("milvus", "run", "standalone")
.WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(2379))
.WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(9000))
.WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(9001))
.DependsOn(etcdContainer)
.DependsOn(minioContainer)
.Build();

await etcdContainer.StartAsync(cancellationToken);
await minioContainer.StartAsync(cancellationToken);
await milvusContainer.StartAsync(cancellationToken);
var container = new MilvusBuilder().Build();

await container.StartAsync(cancellationToken);
// var network = new NetworkBuilder()
// .WithName("milvus-network")
// .Build();
//
// var etcdContainer = new ContainerBuilder()
// .WithImage("quay.io/coreos/etcd:v3.5.5")
// .WithName("milvus-etcd")
// .WithEnvironment("ETCD_AUTO_COMPACTION_MODE", "revision")
// .WithEnvironment("ETCD_AUTO_COMPACTION_RETENTION", "1000")
// .WithEnvironment("ETCD_QUOTA_BACKEND_BYTES", "4294967296")
// .WithEnvironment("ETCD_SNAPSHOT_COUNT", "50000")
// .WithPortBinding(2379, 2379)
// .WithCommand("etcd",
// "-advertise-client-urls=http://0.0.0.0:2379",
// "-listen-client-urls=http://0.0.0.0:2379",
// "--data-dir", "/etcd")
// .Build();
//
// var minioContainer = new ContainerBuilder()
// .WithImage("minio/minio:RELEASE.2023-03-20T20-16-18Z")
// .WithName("milvus-minio")
// .WithPortBinding(9000, 9000)
// .WithPortBinding(9001, 9001)
// .WithEnvironment("MINIO_ACCESS_KEY", "minioadmin")
// .WithEnvironment("MINIO_SECRET_KEY", "minioadmin")
// .WithCommand("minio", "server", "/minio_data", "--console-address", ":9001")
// .Build();
//
// var milvusContainer = new ContainerBuilder()
// .WithImage("milvusdb/milvus:v2.3.0")
// .WithName("milvus-standalone")
// .WithPortBinding(19530, 19530)
// .WithPortBinding(9091, 9091)
// .WithEnvironment("ETCD_ENDPOINTS", "milvus-etcd:2379")
// .WithEnvironment("MINIO_ADDRESS", "milvus-minio:9000")
// .WithCommand("milvus", "run", "standalone")
// .WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(2379))
// .WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(9000))
// .WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(9001))
// .DependsOn(etcdContainer)
// .DependsOn(minioContainer)
// .Build();
//
// await etcdContainer.StartAsync(cancellationToken);
// await minioContainer.StartAsync(cancellationToken);
// await milvusContainer.StartAsync(cancellationToken);

return new DatabaseTestEnvironment
{
VectorDatabase = new MilvusVectorDatabase(new MilvusMemoryStore("localhost")),
Container = milvusContainer,
VectorDatabase = new MilvusVectorDatabase(new MilvusMemoryStore("localhost", port: container.GetMappedPublicPort(19530))),
Container = container,
};
}
case SupportedDatabase.Weaviate:
{
var port1 = Random.Shared.Next(49152, 65535);
var port2 = Random.Shared.Next(49152, 65535);
var container = new ContainerBuilder()
.WithImage("cr.weaviate.io/semitechnologies/weaviate:1.25.10")
.WithPortBinding(hostPort: 8080, containerPort: 8080)
.WithPortBinding(hostPort: 50051, containerPort: 50051)
.WithPortBinding(hostPort: port1, containerPort: 8080)
.WithPortBinding(hostPort: port2, containerPort: 50051)
.WithEnvironment("AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED", "true")
.WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(50051))
.WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(8080))
.Build();

await container.StartAsync(cancellationToken);

await Task.Delay(5000, cancellationToken);

return new DatabaseTestEnvironment
{
VectorDatabase = new WeaviateVectorDatabase(new WeaviateMemoryStore($"http://localhost:8080")),
VectorDatabase = new WeaviateVectorDatabase(new WeaviateMemoryStore($"http://localhost:{port1}")),
Container = container
};
}
Expand Down
2 changes: 1 addition & 1 deletion src/Databases/IntegrationTests/DatabaseTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ await vectorDatabase.Invoking(y => y.GetCollectionAsync(environment.CollectionNa
[TestCase(SupportedDatabase.DuckDb)]
[TestCase(SupportedDatabase.Weaviate)]
//[TestCase(SupportedDatabase.Elasticsearch)]
//[TestCase(SupportedDatabase.Milvus)]
[TestCase(SupportedDatabase.Milvus)]
public async Task AddDocuments_Ok(SupportedDatabase database)
{
await using var environment = await StartEnvironmentForAsync(database);
Expand Down
1 change: 1 addition & 0 deletions src/Databases/IntegrationTests/LangChain.Databases.IntegrationTests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
<ItemGroup>
<PackageReference Include="Testcontainers" />
<PackageReference Include="Testcontainers.Elasticsearch" />
<PackageReference Include="Testcontainers.Milvus" />
<PackageReference Include="Testcontainers.PostgreSql" />
<PackageReference Include="Testcontainers.MongoDb" />
<PackageReference Include="Testcontainers.Redis" />
Expand Down
1 change: 1 addition & 0 deletions src/Directory.Packages.props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
<PackageVersion Include="MSTest.TestAdapter" Version="3.5.1" />
<PackageVersion Include="MSTest.TestFramework" Version="3.5.1" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp.SourceGenerators.Testing.MSTest" Version="1.1.2" />
<PackageVersion Include="Testcontainers.Milvus" Version="3.9.0" />
<PackageVersion Include="Verify.MSTest" Version="26.1.6" />
<PackageVersion Include="Verify.SourceGenerators" Version="2.3.0" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
Expand Down

0 comments on commit 1d08711

Please sign in to comment.