From be550ad23cba28fd99fca724b2de7136f3523a57 Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Thu, 19 Dec 2024 22:12:30 +0100 Subject: [PATCH] Npgsql: Use `EnableDynamicJson` to unlock better container type mappings Vanilla Npgsql provides good enough support to handle CrateDB's ARRAY and OBJECT types better than just plain strings. This patch demonstrates type mappings to native .NET `List` and `Dictionary` types, as well as type mappings to custom .NET POCO types. --- by-language/csharp-npgsql/BasicPoco.cs | 22 +++++ by-language/csharp-npgsql/DemoProgram.cs | 12 ++- by-language/csharp-npgsql/DemoTypes.cs | 80 ++++++++++++------- .../csharp-npgsql/tests/DemoProgramTest.cs | 80 +++++++++++++------ 4 files changed, 141 insertions(+), 53 deletions(-) create mode 100644 by-language/csharp-npgsql/BasicPoco.cs diff --git a/by-language/csharp-npgsql/BasicPoco.cs b/by-language/csharp-npgsql/BasicPoco.cs new file mode 100644 index 00000000..bde7ff96 --- /dev/null +++ b/by-language/csharp-npgsql/BasicPoco.cs @@ -0,0 +1,22 @@ +namespace demo; + +public class BasicPoco +{ + + public string? name { get; set; } + public int? age { get; set; } + + public override bool Equals(object obj) + { + var other = (BasicPoco) obj; + return name == other.name && age == other.age; + } + + public override int GetHashCode() + { + return base.GetHashCode(); + } + + public override string ToString() => "Name: " + name + " Age: " + age; + +} diff --git a/by-language/csharp-npgsql/DemoProgram.cs b/by-language/csharp-npgsql/DemoProgram.cs index b0670c27..edc50749 100644 --- a/by-language/csharp-npgsql/DemoProgram.cs +++ b/by-language/csharp-npgsql/DemoProgram.cs @@ -18,12 +18,18 @@ await Parser.Default.ParseArguments(args) var connString = $"Host={options.Host};Port={options.Port};SSL Mode={options.SslMode};" + $"Username={options.Username};Password={options.Password};Database={options.Database}"; Console.WriteLine($"Connecting to {connString}\n"); - await using var conn = new NpgsqlConnection(connString); - conn.Open(); + + var dataSourceBuilder = new NpgsqlDataSourceBuilder(connString); + dataSourceBuilder.EnableDynamicJson(); + await using var dataSource = dataSourceBuilder.Build(); + await using var conn = dataSource.OpenConnection(); + await DatabaseWorkloads.SystemQueryExample(conn); await DatabaseWorkloads.BasicConversationExample(conn); await DatabaseWorkloads.UnnestExample(conn); - await DatabaseWorkloadsMore.AllTypesExample(conn); + await DatabaseWorkloadsMore.AllTypesNativeExample(conn); + await DatabaseWorkloadsMore.ObjectPocoExample(conn); + await DatabaseWorkloadsMore.ArrayPocoExample(conn); conn.Close(); }); diff --git a/by-language/csharp-npgsql/DemoTypes.cs b/by-language/csharp-npgsql/DemoTypes.cs index 6f2ad062..865426cc 100644 --- a/by-language/csharp-npgsql/DemoTypes.cs +++ b/by-language/csharp-npgsql/DemoTypes.cs @@ -57,9 +57,9 @@ public class AllTypesRecord public class DatabaseWorkloadsMore { - public static async Task AllTypesExample(NpgsqlConnection conn) + public static async Task AllTypesNativeExample(NpgsqlConnection conn) { - Console.WriteLine("Running AllTypesExample"); + Console.WriteLine("Running AllTypesNativeExample"); // Submit DDL, create database schema. await using (var cmd = new NpgsqlCommand("DROP TABLE IF EXISTS testdrive.example", conn)) @@ -154,11 +154,8 @@ INSERT INTO testdrive.example ( cmd.Parameters.AddWithValue("timestamp_tz", "1970-01-02T00:00:00+01:00"); cmd.Parameters.AddWithValue("timestamp_notz", "1970-01-02T00:00:00"); cmd.Parameters.AddWithValue("ip", "127.0.0.1"); - cmd.Parameters.AddWithValue("array", new List{"foo", "bar"}); - // FIXME: System.NotSupportedException: Cannot resolve 'hstore' to a fully qualified datatype name. The datatype was not found in the current database info. - // https://github.com/crate/zk/issues/26 - // cmd.Parameters.AddWithValue("object", new Dictionary(){{"foo", "bar"}}); - cmd.Parameters.AddWithValue("object", """{"foo": "bar"}"""); + cmd.Parameters.AddWithValue("array", NpgsqlDbType.Json, new List{"foo", "bar"}); + cmd.Parameters.AddWithValue("object", NpgsqlDbType.Json, new Dictionary{{"foo", "bar"}}); cmd.Parameters.AddWithValue("geopoint", new List{85.43, 66.23}); // TODO: Check if `GEO_SHAPE` types can be represented by real .NET or Npgsql data types. cmd.Parameters.AddWithValue("geoshape", "POLYGON ((5 5, 10 5, 10 10, 5 10, 5 5))"); @@ -185,20 +182,23 @@ INSERT INTO testdrive.example ( } - public static async Task ContainerTypesExample(NpgsqlConnection conn) + public static async Task ProvisionPoco(NpgsqlConnection conn) { - Console.WriteLine("Running AllTypesExample"); + /*** + * Verify Npgsql POCO mapping with CrateDB. + * https://www.npgsql.org/doc/types/json.html#poco-mapping + */ + Console.WriteLine("Running ProvisionPoco"); // Submit DDL, create database schema. - await using (var cmd = new NpgsqlCommand("DROP TABLE IF EXISTS testdrive.container", conn)) + await using (var cmd = new NpgsqlCommand("DROP TABLE IF EXISTS testdrive.poco", conn)) { cmd.ExecuteNonQuery(); } await using (var cmd = new NpgsqlCommand(""" - CREATE TABLE testdrive.container ( - -- Container types - "array" ARRAY(STRING), + CREATE TABLE testdrive.poco ( + "array" ARRAY(OBJECT(DYNAMIC)), "object" OBJECT(DYNAMIC) ); """, conn)) @@ -208,7 +208,7 @@ CREATE TABLE testdrive.container ( // Insert single data point. await using (var cmd = new NpgsqlCommand(""" - INSERT INTO testdrive.container ( + INSERT INTO testdrive.poco ( "array", "object" ) VALUES ( @@ -217,32 +217,58 @@ INSERT INTO testdrive.container ( ); """, conn)) { - Console.WriteLine(cmd); - // FIXME: While doing conversations with ARRAY types works natively, - // it doesn't work for OBJECT types. - // Yet, they can be submitted as STRING in JSON format. - cmd.Parameters.AddWithValue("array", new List{"foo", "bar"}); - cmd.Parameters.AddWithValue("object", """{"foo": "bar"}"""); + cmd.Parameters.AddWithValue("object", NpgsqlDbType.Json, new BasicPoco { name = "Hotzenplotz" }); + cmd.Parameters.AddWithValue("array", NpgsqlDbType.Json, new List + { + new BasicPoco { name = "Hotzenplotz" }, + new BasicPoco { name = "Petrosilius", age = 42 }, + }); cmd.ExecuteNonQuery(); } // Flush data. - await using (var cmd = new NpgsqlCommand("REFRESH TABLE testdrive.container", conn)) + await using (var cmd = new NpgsqlCommand("REFRESH TABLE testdrive.poco", conn)) { cmd.ExecuteNonQuery(); } + } + + public static async Task ObjectPocoExample(NpgsqlConnection conn) + { + Console.WriteLine("Running ObjectPocoExample"); + + // Provision data. + await ProvisionPoco(conn); + // Query back data. - await using (var cmd = new NpgsqlCommand("SELECT * FROM testdrive.container", conn)) + await using (var cmd = new NpgsqlCommand("SELECT * FROM testdrive.poco", conn)) await using (var reader = cmd.ExecuteReader()) { - var dataTable = new DataTable(); - dataTable.Load(reader); - var payload = JsonConvert.SerializeObject(dataTable); - Console.WriteLine(payload); - return (DataTable) dataTable; + reader.Read(); + var obj = reader.GetFieldValue("object"); + Console.WriteLine(obj); + return obj; } + } + + public static async Task> ArrayPocoExample(NpgsqlConnection conn) + { + Console.WriteLine("Running ArrayPocoExample"); + // Provision data. + await ProvisionPoco(conn); + + // Query back data. + await using (var cmd = new NpgsqlCommand("SELECT * FROM testdrive.poco", conn)) + await using (var reader = cmd.ExecuteReader()) + { + reader.Read(); + var obj = reader.GetFieldValue>("array"); + Console.WriteLine(obj[0]); + Console.WriteLine(obj[1]); + return obj; + } } } diff --git a/by-language/csharp-npgsql/tests/DemoProgramTest.cs b/by-language/csharp-npgsql/tests/DemoProgramTest.cs index c0bfa08c..707bd664 100644 --- a/by-language/csharp-npgsql/tests/DemoProgramTest.cs +++ b/by-language/csharp-npgsql/tests/DemoProgramTest.cs @@ -21,8 +21,11 @@ public DatabaseFixture() CRATEDB_DSN = $"Host=localhost;Port=5432;Username=crate;Password=;Database=testdrive"; } Console.WriteLine($"Connecting to {CRATEDB_DSN}\n"); - Db = new NpgsqlConnection(CRATEDB_DSN); - Db.Open(); + + var dataSourceBuilder = new NpgsqlDataSourceBuilder(CRATEDB_DSN); + dataSourceBuilder.EnableDynamicJson(); + using var dataSource = dataSourceBuilder.Build(); + Db = dataSource.OpenConnection(); } public void Dispose() @@ -83,12 +86,12 @@ public async Task TestUnnestExample() } [Fact] - public async Task TestAllTypesExample() + public async Task TestAllTypesNativeExample() { var conn = fixture.Db; - // Invoke database workload. - var task = DatabaseWorkloadsMore.AllTypesExample(conn); + // Provision data. + var task = DatabaseWorkloadsMore.AllTypesNativeExample(conn); var dt = await task.WaitAsync(TimeSpan.FromSeconds(0.5)); // Check results. @@ -112,12 +115,18 @@ public async Task TestAllTypesExample() Assert.Equal("127.0.0.1", row["ip"]); // Container types - // FIXME: While doing conversations with ARRAY types works natively, - // it doesn't work for OBJECT types. - // Yet, they can be submitted as STRING in JSON format. Assert.Equal(new List{"foo", "bar"}, row["array"]); Assert.Equal("""{"foo":"bar"}""", row["object"]); + // Note: While it works on the ingress side to communicate `Dictionary` types, + // this kind of equality check does not work on the egress side, + // presenting an error that indicates a different internal representation, + // or a programming error ;]. + // + // Expected: [["foo"] = "bar"] + // Actual: {"foo":"bar"} + // Assert.Equal(new Dictionary{{"foo", "bar"}}, row["object"]); + // Geospatial types // TODO: Unlock native data types? // GEO_POINT and GEO_SHAPE types can be marshalled back and forth using STRING. @@ -135,20 +144,12 @@ public async Task TestContainerTypesExample() { var conn = fixture.Db; - // Invoke database workload. - var task = DatabaseWorkloadsMore.ContainerTypesExample(conn); - var dt = await task.WaitAsync(TimeSpan.FromSeconds(0.5)); - - // Check results. - var row = dt.Rows[0]; - // FIXME: While doing conversations with ARRAY types works natively, - // it doesn't work for OBJECT types. - // Yet, they can be submitted as STRING in JSON format. - Assert.Equal(new List{"foo", "bar"}, row["array"]); - Assert.Equal("""{"foo":"bar"}""", row["object"]); + // Provision data. + var task = DatabaseWorkloadsMore.AllTypesNativeExample(conn); + await task.WaitAsync(TimeSpan.FromSeconds(0.5)); - // Run a special query indexing into ARRAY types. - await using (var cmd = new NpgsqlCommand("""SELECT "array[2]" AS foo FROM testdrive.container""", conn)) + // Run an SQL query indexing into ARRAY types. + await using (var cmd = new NpgsqlCommand("""SELECT "array[2]" AS foo FROM testdrive.example""", conn)) await using (var reader = cmd.ExecuteReader()) { var dataTable = new DataTable(); @@ -156,8 +157,8 @@ public async Task TestContainerTypesExample() Assert.Equal("bar", dataTable.Rows[0]["foo"]); } - // Run a special query indexing into OBJECT types. - await using (var cmd = new NpgsqlCommand("""SELECT "object['foo']" AS foo FROM testdrive.container""", conn)) + // Run an SQL query indexing into OBJECT types. + await using (var cmd = new NpgsqlCommand("""SELECT "object['foo']" AS foo FROM testdrive.example""", conn)) await using (var reader = cmd.ExecuteReader()) { var dataTable = new DataTable(); @@ -167,5 +168,38 @@ public async Task TestContainerTypesExample() } + [Fact] + public async Task TestObjectPocoExample() + { + var conn = fixture.Db; + + // Invoke database workload. + var task = DatabaseWorkloadsMore.ObjectPocoExample(conn); + var obj = await task.WaitAsync(TimeSpan.FromSeconds(0.5)); + + // Validate the outcome. + Assert.Equal(new BasicPoco { name = "Hotzenplotz" }, obj); + + } + + [Fact] + public async Task TestArrayPocoExample() + { + var conn = fixture.Db; + + // Invoke database workload. + var task = DatabaseWorkloadsMore.ArrayPocoExample(conn); + var obj = await task.WaitAsync(TimeSpan.FromSeconds(0.5)); + + // Validate the outcome. + var reference = new List + { + new BasicPoco { name = "Hotzenplotz" }, + new BasicPoco { name = "Petrosilius", age = 42 }, + }; + Assert.Equal(reference, obj); + + } + } }