Add SerDe support for C# language.

native-schema-registry/c/CMakeLists.txt

@@ -9,14 +9,15 @@ set(CMAKE_C_STANDARD 99)
 ##Global variables
 #Path to GraalVM generated shared library
 set(LIB_NATIVE_SCHEMA_REGISTRY_PATH "${PROJECT_SOURCE_DIR}/../target")
-
+set(LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME_PREFIX libnativeschemaregistry)
 ##OS Specific variables
 IF (WIN32)
-    set(LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME libnativeschemaregistry.dll)
+    set(LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME ${LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME_PREFIX}.dll)
+    set(LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_OBJ_NAME ${LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME_PREFIX}.lib)
 ELSEIF(APPLE)
-    set(LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME libnativeschemaregistry.dylib)
+    set(LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME ${LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME_PREFIX}.dylib)
 ELSE()
-    set(LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME libnativeschemaregistry.so)
+    set(LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME ${LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME_PREFIX}.so)
 ENDIF()
 
 add_subdirectory("src")

native-schema-registry/c/src/CMakeLists.txt

@@ -1,57 +1,88 @@
-set(module_name native_schema_registry_c)
+set(DATA_TYPES_MODULE_NAME native_schema_registry_c_data_types)
+set(SERDE_MODULE_NAME native_schema_registry_c)
+set(NATIVE_SCHEMA_REGISTRY_MODULE_NAME libnativeschemaregistry)
+set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 
 #Fix the include path in GraalVM generated header file.
 execute_process(COMMAND sed -ie "s/<graal_isolate.h>/\"graal_isolate.h\"/" ${LIB_NATIVE_SCHEMA_REGISTRY_PATH}/libnativeschemaregistry.h)
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftest-coverage -fprofile-arcs -ggdb3 -O2 -Wall")
 
+#Adding modules in the build order.
 add_library(
-        ${module_name} SHARED
+        ${DATA_TYPES_MODULE_NAME} SHARED
         glue_schema_registry_schema.c
         read_only_byte_array.c
         mutable_byte_array.c
+)
+
+#Add reference to the GraalVM generated Native schema registry module.
+add_library(
+        ${NATIVE_SCHEMA_REGISTRY_MODULE_NAME} SHARED
+        IMPORTED
+)
+set_target_properties(
+        ${NATIVE_SCHEMA_REGISTRY_MODULE_NAME}
+        PROPERTIES
+        IMPORTED_LOCATION "${LIB_NATIVE_SCHEMA_REGISTRY_PATH}/${LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME}"
+        IMPORTED_IMPLIB "${LIB_NATIVE_SCHEMA_REGISTRY_PATH}/${LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_OBJ_NAME}"
+        INTERFACE_INCLUDE_DIRECTORIES "${LIB_NATIVE_SCHEMA_REGISTRY_PATH}"
+)
+
+add_library(
+        ${SERDE_MODULE_NAME} SHARED
         glue_schema_registry_serializer.c
         glue_schema_registry_deserializer.c
-        )
+)
 
 target_link_libraries(
-        ${module_name}
-        ${LIB_NATIVE_SCHEMA_REGISTRY_PATH}/${LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME}
-)
-
-#set(CMAKE_SWIG_FLAGS)
-#find_package(SWIG REQUIRED)
-#include(UseSWIG)
-#
-#file(
-#    GLOB GsrSerDeSrc
-#    "swig/*.i"
-#)
-
-##Swig language specific config. TODO: Move to different files
-#Begin Python
-#set(PYTHON_PROJECT_DIR, CPy)
-#TODO: Below Swig target will be built on the a Python manylinux Docker images
-#set(PYTHON_INCLUDE_PATH "")
-#
-#swig_add_library(
-#        GsrSerDe
-#        TYPE SHARED
-#        LANGUAGE python
-#        OUTPUT_DIR CPy/GsrSerDe
-#        SOURCES ${GsrSerDeSrc}
-#)
-
-#set_target_properties(
-#        GsrSerDe
-#        PROPERTIES
-#        INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/clang/include/;${PYTHON_INCLUDE_PATH}"
-#        SWIG_USE_TARGET_INCLUDE_DIRECTORIES TRUE
-#)
-
-#target_link_libraries(
-#        GsrSerDe
-#        PUBLIC
-#        ${module_name}
-#)
-#End Python
+        ${SERDE_MODULE_NAME}
+        ${NATIVE_SCHEMA_REGISTRY_MODULE_NAME}
+)
+
+##Swig compilation
+set(CMAKE_SWIG_FLAGS)
+find_package(SWIG REQUIRED)
+include(UseSWIG)
+
+file(
+        GLOB GsrSerDeSrc
+        "swig/glue_schema_registry_serde.i"
+)
+
+#####CSharp
+set(CSHARP_MODULE_NAME GsrSerDeCsGen)
+set(CSHARP_SOURCE_PATH ${PROJECT_SOURCE_DIR}/../csharp/AWSGsrSerDe/AWSGsrSerDe/)
+set(CSHARP_GEN_LIB_PATH ${CSHARP_SOURCE_PATH}/bin/)
+
+swig_add_library(
+        ${CSHARP_MODULE_NAME}
+        TYPE SHARED
+        LANGUAGE csharp
+        OUTPUT_DIR "${CSHARP_SOURCE_PATH}"
+        SOURCES ${GsrSerDeSrc}
+)
+set_target_properties(
+        ${CSHARP_MODULE_NAME}
+        PROPERTIES
+        INCLUDE_DIRECTORIES "${PROJECT_SOURCE_DIR}/include/"
+        SWIG_USE_TARGET_INCLUDE_DIRECTORIES TRUE
+)
+swig_link_libraries(
+        ${CSHARP_MODULE_NAME}
+        PUBLIC
+        ${DATA_TYPES_MODULE_NAME}
+        ${SERDE_MODULE_NAME}
+        ${NATIVE_SCHEMA_REGISTRY_MODULE_NAME}
+)
+##TODO: Fix this during release. We should segregate Debug and Release
+##Copying built libraries to CSharp project
+add_custom_command(
+        TARGET ${CSHARP_MODULE_NAME}
+        POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${CSHARP_MODULE_NAME}> ${CSHARP_GEN_LIB_PATH}/
+        COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${DATA_TYPES_MODULE_NAME}> ${CSHARP_GEN_LIB_PATH}/
+        COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${NATIVE_SCHEMA_REGISTRY_MODULE_NAME}> ${CSHARP_GEN_LIB_PATH}/
+        COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:${SERDE_MODULE_NAME}> ${CSHARP_GEN_LIB_PATH}/
+)
+

native-schema-registry/c/src/swig/glue_schema_registry_deserializer.i

@@ -0,0 +1,24 @@
+%module GsrSerDe
+%{
+#include "glue_schema_registry_deserializer.h"
+%}
+
+typedef struct glue_schema_registry_deserializer {
+
+    %newobject decode_schema(read_only_byte_array *array);
+    #if defined(SWIGCSHARP)
+      %newobject decode(read_only_byte_array *array);
+    #endif
+
+    %extend {
+        glue_schema_registry_deserializer();
+
+        ~glue_schema_registry_deserializer();
+
+        mutable_byte_array *decode(read_only_byte_array *array);
+
+        bool can_decode(read_only_byte_array *array);
+
+        glue_schema_registry_schema* decode_schema(read_only_byte_array *array);
+    }
+} glue_schema_registry_deserializer;

native-schema-registry/c/src/swig/glue_schema_registry_serde.i

@@ -0,0 +1,8 @@
+%module GsrSerDe
+%{
+%}
+%include "read_only_byte_array.i"
+%include "mutable_byte_array.i"
+%include "glue_schema_registry_schema.i"
+%include "glue_schema_registry_serializer.i"
+%include "glue_schema_registry_deserializer.i"

native-schema-registry/c/src/swig/glue_schema_registry_serializer.i

@@ -4,6 +4,10 @@
 
 %}
 typedef struct glue_schema_registry_serializer {
+    #if defined(SWIGCSHARP)
+        //Pass the ownership to C# runtime.
+        %newobject encode(read_only_byte_array *array, const char *transport_name, glue_schema_registry_schema *gsr_schema);
+    #endif
     %extend {
         glue_schema_registry_serializer();
 

native-schema-registry/c/src/swig/mutable_byte_array.i

@@ -3,6 +3,10 @@
 #include "mutable_byte_array.h"
 %}
 
+#if defined(SWIGCSHARP)
+%include "arrays_csharp.i"
+#endif
+
 #if defined(SWIGPYTHON)
 //Converts the unsigned char * to a Python Bytes object.
 %typemap(out) mutable_byte_array * %{
@@ -22,6 +26,16 @@ typedef struct mutable_byte_array {
         mutable_byte_array(size_t len);
 
         ~mutable_byte_array();
+        //Uses array output typemap that copies the mutable_byte_array contents
+        //into given array.
+        //TODO: Optimization: We can avoid copying large data by figuring out a way
+        //to expose underlying buffers.
+        #if defined(SWIGCSHARP)
+            %apply unsigned char OUTPUT[] {unsigned char *data};
+        #endif
+        void get_data_copy(unsigned char *data) {
+            memcpy(data, $self->data, $self->max_len);
+        }
 
         unsigned char * get_data();
 

native-schema-registry/c/src/swig/read_only_byte_array.i

@@ -7,11 +7,19 @@
 %include "pybuffer.i"
 #endif
 
+#if defined(SWIGCSHARP)
+%include "arrays_csharp.i"
+#endif
+
 typedef struct read_only_byte_array {
     %extend {
 //PyBuffer provides type-maps for accepting Python bytes.
 #if defined(SWIGPYTHON)
         %pybuffer_binary(unsigned char * data, size_t len);
+#endif
+#if defined(SWIGCSHARP)
+        //Typemap that converts the C# byte [] into unsigned char *
+        %apply unsigned char INPUT[] {unsigned char *data};
 #endif
         read_only_byte_array(unsigned char *data, size_t len);
 

native-schema-registry/c/test/CMakeLists.txt

@@ -18,6 +18,7 @@ foreach (test ${tests})
             )
     target_link_libraries(
             "${test}"
+            native_schema_registry_c_data_types
             native_schema_registry_c
             ${LIB_NATIVE_SCHEMA_REGISTRY_PATH}/${LIB_NATIVE_SCHEMA_REGISTRY_LIBRARY_NAME}
             cmocka

native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/AWSGsrSerDe.Tests.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>netcoreapp3.1</TargetFramework>
+        <IsPackable>false</IsPackable>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <PackageReference Include="NUnit" Version="3.12.0" />
+        <PackageReference Include="NUnit3TestAdapter" Version="3.16.1" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\AWSGsrSerDe\AWSGsrSerDe.csproj" />
+    </ItemGroup>
+
+</Project>

native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.Tests/GlueSchemaRegistryTests.cs

@@ -0,0 +1,79 @@
+using NUnit.Framework;
+using System.IO;
+using Avro;
+using Avro.Generic;
+using Avro.IO;
+
+namespace AWSGsrSerDe.Tests
+{
+    [TestFixture]
+    public class BasicSerDeTest
+    {
+
+        private GlueSchemaRegistrySerializer _serializer;
+        private GlueSchemaRegistryDeserializer _deserializer;
+        private GlueSchemaRegistrySchema _schema;
+
+        private const string TestAvroSchema = "{\"namespace\": \"example.avro\",\n"
+                                              + " \"type\": \"record\",\n"
+                                              + " \"name\": \"User\",\n"
+                                              + " \"fields\": [\n"
+                                              + "     {\"name\": \"name\", \"type\": \"string\"},\n"
+                                              + "     {\"name\": \"favorite_number\",  \"type\": [\"int\", \"null\"]},\n"
+                                              + "     {\"name\": \"favorite_color\", \"type\": [\"string\", \"null\"]}\n"
+                                              + " ]\n"
+                                              + "}";
+
+        private byte[] _data;
+
+        [SetUp]
+        public void Setup()
+        {
+            _serializer = new GlueSchemaRegistrySerializer();
+            _deserializer = new GlueSchemaRegistryDeserializer();
+
+            _schema = new GlueSchemaRegistrySchema("TestSchemaName", TestAvroSchema, "AVRO");
+            _data = GetAvroMessage();
+        }
+
+        [Test]
+        public void AssertSerDe()
+        {
+            var output = _serializer.Encode("TestTransportName", _schema, _data);
+
+            Assert.IsNotEmpty(output);
+            Assert.True(_deserializer.CanDecode(output));
+
+            var decoded = _deserializer.Decode(output);
+            var deserializedSchema = _deserializer.DecodeSchema(output);
+
+            Assert.AreEqual(_schema, deserializedSchema);
+
+            Assert.IsNotEmpty(decoded);
+            Assert.AreEqual(_data.Length, decoded.Length);
+
+            for (var index = 0; index < _data.Length; index++)
+            {
+                Assert.AreEqual(_data[index], decoded[index]);
+            }
+        }
+
+        private static byte[] GetAvroMessage()
+        {
+            var recordSchema = Schema.Parse(TestAvroSchema);
+            var user = new GenericRecord((RecordSchema)recordSchema);
+
+            user.Add ("name", "Alyssa");
+            user.Add("favorite_number", 256);
+            user.Add("favorite_color", "blue");
+
+            var genericDatumWriter = new  GenericDatumWriter<GenericRecord>(recordSchema);
+            var encoded = new MemoryStream();
+            var encoder = new BinaryEncoder(encoded);
+            genericDatumWriter.Write(user, encoder);
+            encoder.Flush();
+
+            return encoded.ToArray();
+        }
+    }
+}

native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe.sln

@@ -0,0 +1,22 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWSGsrSerDe", "AWSGsrSerDe\AWSGsrSerDe.csproj", "{A36E4C3C-B7C4-4670-90B1-2FC9D883CCD0}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AWSGsrSerDe.Tests", "AWSGsrSerDe.Tests\AWSGsrSerDe.Tests.csproj", "{AD987D8A-EC56-47E8-837F-4D2A9E4ABA6C}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{A36E4C3C-B7C4-4670-90B1-2FC9D883CCD0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A36E4C3C-B7C4-4670-90B1-2FC9D883CCD0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A36E4C3C-B7C4-4670-90B1-2FC9D883CCD0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A36E4C3C-B7C4-4670-90B1-2FC9D883CCD0}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AD987D8A-EC56-47E8-837F-4D2A9E4ABA6C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AD987D8A-EC56-47E8-837F-4D2A9E4ABA6C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AD987D8A-EC56-47E8-837F-4D2A9E4ABA6C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AD987D8A-EC56-47E8-837F-4D2A9E4ABA6C}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal

native-schema-registry/csharp/AWSGsrSerDe/AWSGsrSerDe/AWSGsrSerDe.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>netcoreapp3.1</TargetFramework>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Apache.Avro" Version="1.11.0" />
+    </ItemGroup>
+
+</Project>