Skip to content

Commit

Permalink
feat: Use mutations when possible (#72)
Browse files Browse the repository at this point in the history
* feat: use mutations when possible

* feat: support concurrency tokens with mutations

* feat: enable retries for mutations + add tests

* docs: add comments

* fix: the emulator now supports computed columns

The emulator now supports computed columns, as long as the computed
column is the last column in the table.

* test: move mutation tests to separate class

* docs: add sample for mutation usage

* test: add integration test for reading mutations

* fix: remove commented code

* fix: propagation of computed valules did not work with manual transactions

* fix: only use transaction for propagation if manual transactions are used

* fix: add additional documentation + fix clone method + use auto property

* fix: address review comments
  • Loading branch information
olavloite authored Apr 2, 2021
1 parent 691a29d commit a50adb8
Show file tree
Hide file tree
Showing 28 changed files with 1,319 additions and 249 deletions.
8 changes: 2 additions & 6 deletions .github/workflows/build-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,19 +40,15 @@ jobs:
working-directory: ./Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests
# The emulator requires the integration tests to be run serially, so we temporarily
# overwrite the default test configuration.
run: |
echo "{ \"parallelizeTestCollections\": false }" > xunit.runner.json
dotnet test --verbosity normal
run: dotnet test --verbosity normal
env:
JOB_TYPE: test
SPANNER_EMULATOR_HOST: localhost:9010
TEST_PROJECT: emulator-test-project
TEST_SPANNER_INSTANCE: test-instance
- name: Integration Tests on Production
working-directory: ./Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests
run: |
echo "{ \"parallelizeTestCollections\": false }" > xunit.runner.json
dotnet test --verbosity normal
run: dotnet test --verbosity normal
env:
JOB_TYPE: test
TEST_PROJECT: ${{ secrets.GCP_PROJECT_ID }}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
using BenchmarkDotNet.Running;
using Google.Api.Gax;
using Google.Cloud.EntityFrameworkCore.Spanner.Extensions;
using Google.Cloud.EntityFrameworkCore.Spanner.Infrastructure;
using Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests;
using Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.Model;
using Google.Cloud.EntityFrameworkCore.Spanner.Storage;
Expand All @@ -36,10 +37,17 @@ internal class BenchmarkSampleDbContext : SpannerSampleDbContext

private readonly string _connectionString;

internal BenchmarkSampleDbContext(bool useRealSpanner, string connectionString) : base()
private readonly MutationUsage _mutationUsage;

internal BenchmarkSampleDbContext(bool useRealSpanner, string connectionString) : this(useRealSpanner, connectionString, MutationUsage.ImplicitTransactions)
{
}

internal BenchmarkSampleDbContext(bool useRealSpanner, string connectionString, MutationUsage mutationUsage) : base()
{
_useRealSpanner = useRealSpanner;
_connectionString = connectionString;
_mutationUsage = mutationUsage;
}

protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
Expand All @@ -49,6 +57,7 @@ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
var connection = _useRealSpanner ? new SpannerConnection(_connectionString) : new SpannerConnection(_connectionString, ChannelCredentials.Insecure);
optionsBuilder
.UseSpanner(connection)
.UseMutations(_mutationUsage)
.UseLazyLoadingProxies();
}
}
Expand Down Expand Up @@ -271,13 +280,54 @@ public int SaveOneRowWithFetchAfterEF()

[Benchmark]
public long SaveMultipleRowsSpanner()
{
using var connection = CreateConnection();
var singerId = MaybeCreateSingerSpanner(connection);
return connection.RunWithRetriableTransaction(transaction =>
{
var updateCount = 0;
for (int row = 0; row < 100; row++)
{
var command = connection.CreateInsertCommand("Albums", new SpannerParameterCollection
{
new SpannerParameter("id", SpannerDbType.Int64, _useRealSpanner ? _fixture.RandomLong() : row),
new SpannerParameter("title", SpannerDbType.String, "Pete"),
new SpannerParameter("releaseDate", SpannerDbType.Date, new DateTime(1998, 10, 6)),
new SpannerParameter("singerId", SpannerDbType.Int64, singerId),
});
updateCount += command.ExecuteNonQuery();
}
return updateCount;
});
}

[Benchmark]
public long SaveMultipleRowsEF()
{
using var db = new BenchmarkSampleDbContext(_useRealSpanner, _connectionString);
var singerId = MaybeCreateSingerEF(db);
for (int row = 0; row < 100; row++)
{
db.Albums.Add(new Albums
{
AlbumId = _useRealSpanner ? _fixture.RandomLong() : row,
Title = "Pete",
ReleaseDate = new SpannerDate(1998, 10, 6),
SingerId = singerId,
});
}
return db.SaveChanges();
}

[Benchmark]
public long SaveMultipleRowsUsingDmlSpanner()
{
using var connection = CreateConnection();
var singerId = MaybeCreateSingerSpanner(connection);
return connection.RunWithRetriableTransaction(transaction =>
{
var command = transaction.CreateBatchDmlCommand();
for (int row = 0; row < 10; row++)
for (int row = 0; row < 100; row++)
{
command.Add("INSERT INTO Albums (AlbumId, Title, ReleaseDate, SingerId) VALUES (@id, @title, @releaseDate, @singerId)", new SpannerParameterCollection
{
Expand All @@ -292,11 +342,11 @@ public long SaveMultipleRowsSpanner()
}

[Benchmark]
public long SaveMultipleRowsEF()
public long SaveMultipleRowsUsingDmlEF()
{
using var db = new BenchmarkSampleDbContext(_useRealSpanner, _connectionString);
using var db = new BenchmarkSampleDbContext(_useRealSpanner, _connectionString, MutationUsage.Never);
var singerId = MaybeCreateSingerEF(db);
for (int row = 0; row < 10; row++)
for (int row = 0; row < 100; row++)
{
db.Albums.Add(new Albums
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@
using Google.Cloud.Spanner.Common.V1;
using Google.Cloud.Spanner.V1.Internal.Logging;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.ChangeTracking;
using Microsoft.EntityFrameworkCore.ValueGeneration;

namespace Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -446,7 +446,7 @@ public void ShouldThrowLengthValidationException()
"this is too long string should throw length validation error"
});

Assert.Throws<SpannerBatchNonQueryException>(() => context.SaveChanges());
Assert.Throws<SpannerException>(() => context.SaveChanges());
}

[Fact]
Expand All @@ -463,7 +463,7 @@ public void ShouldThrowRequiredFieldValidationException()
}
});

Assert.Throws<SpannerBatchNonQueryException>(() => context.SaveChanges());
Assert.Throws<SpannerException>(() => context.SaveChanges());
}

[Fact]
Expand Down Expand Up @@ -521,7 +521,7 @@ public void ShouldThrowInterleaveTableOnInsert()
PublishDate = new DateTime(2020, 12, 1)
};
context.Articles.Add(article);
Assert.Throws<SpannerBatchNonQueryException>(() => context.SaveChanges());
Assert.Throws<SpannerException>(() => context.SaveChanges());
}

[Fact]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
// limitations under the License.

using Google.Api.Gax;
using Google.Cloud.EntityFrameworkCore.Spanner.Infrastructure;
using Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.Model;
using Google.Cloud.Spanner.Common.V1;
using Google.Cloud.Spanner.V1.Internal.Logging;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public async Task SaveChangesIsAtomic()
SingerId = invalidSingerId, // Invalid, does not reference an actual Singer
Title = "Some title",
});
await Assert.ThrowsAsync<SpannerBatchNonQueryException>(() => db.SaveChangesAsync());
await Assert.ThrowsAsync<SpannerException>(() => db.SaveChangesAsync());
}

using (var db = new TestSpannerSampleDbContext(_fixture.DatabaseName))
Expand Down Expand Up @@ -298,6 +298,58 @@ public void TransactionRetry(bool disableInternalRetries)
);
}

[Fact]
public async Task ComputedColumnIsPropagatedInManualTransaction()
{
using var db = new TestSpannerSampleDbContext(_fixture.DatabaseName);
using var transaction = await db.Database.BeginTransactionAsync();
var id = _fixture.RandomLong();
db.Singers.Add(new Singers
{
SingerId = id,
FirstName = "Alice",
LastName = "Ferguson",
});
await db.SaveChangesAsync();

var row = await db.Singers.FindAsync(id);
Assert.Equal("Alice Ferguson", row.FullName);

await transaction.CommitAsync();
}

[Fact]
public async Task ManualTransactionCannotReadMutations()
{
var options = new DbContextOptionsBuilder<SpannerSampleDbContext>()
.UseSpanner(_fixture.ConnectionString)
.UseMutations(Infrastructure.MutationUsage.Always)
.Options;
using var db = new SpannerSampleDbContext((DbContextOptions<SpannerSampleDbContext>)options);
using var transaction = await db.Database.BeginTransactionAsync();
var id = _fixture.RandomLong();
db.TableWithAllColumnTypes.Add(new TableWithAllColumnTypes
{
ColInt64 = id,
ColString = "Test row",
});
await db.SaveChangesAsync();

// Getting the row from the context using its id should work, as the row is attached to the context.
var row = await db.TableWithAllColumnTypes.FindAsync(id);
Assert.NotNull(row);

// Getting the row by querying will not work, as the context is using mutations for all transactions,
// and mutations are not readable during the same transaction.
row = await db.TableWithAllColumnTypes.Where(record => record.ColInt64 == id).FirstOrDefaultAsync();
Assert.Null(row);

// Commit the transaction. The row should now be readable through a query.
await transaction.CommitAsync();
row = await db.TableWithAllColumnTypes.Where(record => record.ColInt64 == id).FirstOrDefaultAsync();
Assert.NotNull(row);
}

private async Task InsertRandomSinger(bool disableInternalRetries)
{
var rnd = new Random(Guid.NewGuid().GetHashCode());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ public SpannerSampleDbContext(string connectionString, DbContextOptions<SpannerS
public virtual DbSet<Performance> Performances { get; set; }

protected override void OnConfiguring(DbContextOptionsBuilder options)
// Configure Entity Framework to use a Cloud Spanner database.
=> options.UseSpanner(_connectionString);

protected override void OnModelCreating(ModelBuilder modelBuilder)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
// Copyright 2021 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

using Google.Cloud.EntityFrameworkCore.Spanner.Infrastructure;
using Google.Cloud.EntityFrameworkCore.Spanner.Samples.SampleModel;
using Microsoft.EntityFrameworkCore;
using System;
using System.Threading.Tasks;

/// <summary>
/// By default the Cloud Spanner Entity Framework Core provider will use mutations for
/// updates that are executed in an implicit transaction, and DML for updates that are
/// executed in a manual transaction. The reason for this default behavior is that:
/// 1. Mutations are more efficient for multiple small insert/update/delete statements.
/// Mutations do not support read-your-writes semantics. The lack of this feature is
/// not a problem for implicit transactions, as it is impossible to execute queries
/// in an implicit transaction.
/// 2. DML statements are less efficient than mutations for multiple small updates, but
/// they do support read-your-writes semantics. Manual transactions can span multiple
/// statements and queries, and the lack of read-your-writes would negatively impact
/// the usefulness of manual transactions.
///
/// An application can configure a DbContext to use mutations or DML statements for all
/// updates, instead of using one type for implicit transactions and the other for manual
/// transactions. Changing the default behavior will have the following impact on your
/// application:
/// 1. Configuring a DbContext to always use Mutations: This will speed up the execution
/// speed of large batches of inserts/updates/deletes, but it will also mean that the
/// application will not be able to read its own writes during a manual transaction.
/// 2. Configuring a DbContext to always use DML: This will reduce the execution speed of
/// large batches of inserts/updates/deletes that are executed as implicit transactions.
///
/// Run from the command line with `dotnet run MutationUsageSample`
/// </summary>
public static class MutationUsageSample
{
/// <summary>
/// A sample DbContext that supports manual configuration of when to use mutations.
/// </summary>
internal class SpannerSampleMutationUsageDbContext : SpannerSampleDbContext
{
internal MutationUsage MutationUsage { get; }

internal SpannerSampleMutationUsageDbContext(string connectionString, MutationUsage mutationUsage) : base(connectionString)
{
MutationUsage = mutationUsage;
}

protected override void OnConfiguring(DbContextOptionsBuilder options)
{
base.OnConfiguring(options);
options.UseMutations(MutationUsage);
}
}

public static async Task Run(string connectionString)
{
// Create a DbContext that uses Mutations for implicit transactions. This is the default behavior of a DbContext.
// This means that the context will use mutations for implicit transactions, and DML for manual transactions.
await RunSampleStatements(new SpannerSampleMutationUsageDbContext(connectionString, MutationUsage.ImplicitTransactions));

// Create a DbContext that always uses Mutations. This means that read-your-writes will be disabled for manual
// transactions. It also means that large batches of inserts/updates/deletes will execute faster. It is recommended
// to use this mode for manual transactions that do not need read-your-writes, and that do contain larege update batches.
await RunSampleStatements(new SpannerSampleMutationUsageDbContext(connectionString, MutationUsage.Always));

// Create a DbContext that never uses Mutations. All inserts/updates/deletes will be executed as DML statements.
await RunSampleStatements(new SpannerSampleMutationUsageDbContext(connectionString, MutationUsage.Never));
}

private static async Task RunSampleStatements(SpannerSampleMutationUsageDbContext context)
{
Console.WriteLine();
Console.WriteLine($"Running sample with mutation usage {context.MutationUsage}");
// Add a new singer using an implicit transaction.
var singerId = Guid.NewGuid();
await context.Singers.AddAsync(new Singer
{
SingerId = singerId,
FirstName = "Bernhard",
LastName = "Bennet"
});
var count = await context.SaveChangesAsync();
Console.WriteLine($"Added {count} singer in an implicit transaction with a context that has mutation usage {context.MutationUsage}.");

// Now try to read the row back using the same context. This will always return true.
var exists = await context.Singers
.FromSqlInterpolated($"SELECT * FROM Singers WHERE SingerId={singerId}")
.FirstOrDefaultAsync();
Console.WriteLine($"Can read singer after implicit transaction: {exists != null}");

// Start a read/write transaction that will be used with the database context.
using var transaction = await context.Database.BeginTransactionAsync();

// Create a new Singer, add it to the context and save the changes.
// These changes have not yet been committed to the database and are
// therefore not readable for other processes. It will be readable for
// the same transaction, unless MutationUsage has been set to Always.
singerId = Guid.NewGuid();
await context.Singers.AddAsync(new Singer
{
SingerId = singerId,
FirstName = "Alice",
LastName = "Wendelson"
});
count = await context.SaveChangesAsync();
Console.WriteLine($"Added {count} singer in a manual transaction with a context that has mutation usage {context.MutationUsage}.");

// Now try to read the row back using the same context. This will return true for contexts that use
// DML for manual transactions.
exists = await context.Singers
.FromSqlInterpolated($"SELECT * FROM Singers WHERE SingerId={singerId}")
.FirstOrDefaultAsync();
Console.WriteLine($"Can read singer inside transaction: {exists != null}");

// Commit the transaction. The singer is now always readable.
await transaction.CommitAsync();

exists = await context.Singers
.FromSqlInterpolated($"SELECT * FROM Singers WHERE SingerId={singerId}")
.FirstOrDefaultAsync();
Console.WriteLine($"Can read singer after commit: {exists != null}");
}
}
Loading

0 comments on commit a50adb8

Please sign in to comment.