From 263b206159a8388f74e40cdbf41074d170086efd Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 9 Oct 2024 15:24:34 +0530 Subject: [PATCH 01/35] Fix java 17 dependencies in Ballerina.toml --- build-config/resources/Ballerina.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-config/resources/Ballerina.toml b/build-config/resources/Ballerina.toml index c5e7203..6caa187 100644 --- a/build-config/resources/Ballerina.toml +++ b/build-config/resources/Ballerina.toml @@ -33,13 +33,13 @@ artifactId = "jackson-core" version = "@jackson.version@" path = "./lib/jackson-core-@jackson.version@.jar" -[[platform.java11.dependency]] +[[platform.java17.dependency]] groupId = "com.fasterxml.jackson.core" artifactId = "jackson-annotations" version = "@jackson.version@" path = "./lib/jackson-annotations-@jackson.version@.jar" -[[platform.java11.dependency]] +[[platform.java17.dependency]] groupId = "com.fasterxml.jackson.core" artifactId = "jackson-databind" version = "@jackson.version@" From f1c96f453893440e2a52d9a75c6346cacc7f5816 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 9 Oct 2024 15:24:49 +0530 Subject: [PATCH 02/35] Update avro, jackson dependencies --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 9d11178..b45ad16 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,5 +14,5 @@ ballerinaGradlePluginVersion=2.0.1 # Dependencies stdlibIoVersion=1.6.0 -avroVersion=1.11.3 -jacksonVersion=2.17.0 +avroVersion=1.11.4 +jacksonVersion=2.18.0 From bd0f2fff3ba36d775b16697cc9c7c02dcb3e2362 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 9 Oct 2024 15:27:00 +0530 Subject: [PATCH 03/35] Update the lang version to 2201.10.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index b45ad16..0bf9db1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ org.gradle.caching=true group=io.ballerina.lib version=1.0.1-SNAPSHOT -ballerinaLangVersion=2201.9.0 +ballerinaLangVersion=2201.10.0 checkstylePluginVersion=10.12.0 spotbugsPluginVersion=5.0.14 From bcb66d120076a7ce78673f3eaa8e3d12835cda1f Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 9 Oct 2024 15:27:45 +0530 Subject: [PATCH 04/35] Bump the avro version to 1.1.1 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 0bf9db1..a23fc64 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.caching=true group=io.ballerina.lib -version=1.0.1-SNAPSHOT +version=1.1.1-SNAPSHOT ballerinaLangVersion=2201.10.0 checkstylePluginVersion=10.12.0 From 60fc532ddeeaa1a6d4d28e4adcf6f21139fec71b Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 9 Oct 2024 15:28:05 +0530 Subject: [PATCH 05/35] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 26 +++++++++++++------------- ballerina/Dependencies.toml | 6 +++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 80ba45f..ebafe5e 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "1.0.0" +version = "1.1.1" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] @@ -18,29 +18,29 @@ graalvmCompatible = true [[platform.java17.dependency]] groupId = "io.ballerina.lib" artifactId = "avro-native" -version = "1.0.0" -path = "../native/build/libs/avro-native-1.0.0.jar" +version = "1.1.1" +path = "../native/build/libs/avro-native-1.1.1-SNAPSHOT.jar" [[platform.java17.dependency]] groupId = "org.apache.avro" artifactId = "avro" -version = "1.11.3" -path = "./lib/avro-1.11.3.jar" +version = "1.11.4" +path = "./lib/avro-1.11.4.jar" [[platform.java17.dependency]] groupId = "com.fasterxml.jackson.core" artifactId = "jackson-core" -version = "2.17.0" -path = "./lib/jackson-core-2.17.0.jar" +version = "2.18.0" +path = "./lib/jackson-core-2.18.0.jar" -[[platform.java11.dependency]] +[[platform.java17.dependency]] groupId = "com.fasterxml.jackson.core" artifactId = "jackson-annotations" -version = "2.17.0" -path = "./lib/jackson-annotations-2.17.0.jar" +version = "2.18.0" +path = "./lib/jackson-annotations-2.18.0.jar" -[[platform.java11.dependency]] +[[platform.java17.dependency]] groupId = "com.fasterxml.jackson.core" artifactId = "jackson-databind" -version = "2.17.0" -path = "./lib/jackson-databind-2.17.0.jar" +version = "2.18.0" +path = "./lib/jackson-databind-2.18.0.jar" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index e770f53..08477a5 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -5,12 +5,12 @@ [ballerina] dependencies-toml-version = "2" -distribution-version = "2201.9.0" +distribution-version = "2201.10.0" [[package]] org = "ballerina" name = "avro" -version = "1.0.0" +version = "1.1.1" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"}, @@ -23,7 +23,7 @@ modules = [ [[package]] org = "ballerina" name = "io" -version = "1.6.0" +version = "1.6.1" scope = "testOnly" dependencies = [ {org = "ballerina", name = "jballerina.java"}, From 40230f119dc91929bd174e49b0c4afc84d03c407 Mon Sep 17 00:00:00 2001 From: Nuvindu Nirmana <63797478+Nuvindu@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:32:15 +0530 Subject: [PATCH 06/35] Update the avro version to 1.1.0 --- ballerina/Ballerina.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index ebafe5e..8a7ba79 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "1.1.1" +version = "1.1.0" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] From d271540bf7a78f612fad7c1e41e64c5ccf5cb3c1 Mon Sep 17 00:00:00 2001 From: Nuvindu Nirmana <63797478+Nuvindu@users.noreply.github.com> Date: Wed, 9 Oct 2024 15:35:05 +0530 Subject: [PATCH 07/35] Update the lang version to 2201.9.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index a23fc64..4791a28 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ org.gradle.caching=true group=io.ballerina.lib version=1.1.1-SNAPSHOT -ballerinaLangVersion=2201.10.0 +ballerinaLangVersion=2201.9.0 checkstylePluginVersion=10.12.0 spotbugsPluginVersion=5.0.14 From d26a1ec9a4ff55fa12b986b0f9434a6aefc4161b Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 9 Oct 2024 15:39:19 +0530 Subject: [PATCH 08/35] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 2 +- ballerina/Dependencies.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 8a7ba79..ebafe5e 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "1.1.0" +version = "1.1.1" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 08477a5..06bc9d9 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -5,7 +5,7 @@ [ballerina] dependencies-toml-version = "2" -distribution-version = "2201.10.0" +distribution-version = "2201.9.0" [[package]] org = "ballerina" From 0d955ecc2260e9106cc739e0b1fade088a686d2c Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 9 Oct 2024 15:44:57 +0530 Subject: [PATCH 09/35] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 6 +++--- ballerina/Dependencies.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index ebafe5e..2ab57cd 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "1.1.1" +version = "1.1.0" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] @@ -18,8 +18,8 @@ graalvmCompatible = true [[platform.java17.dependency]] groupId = "io.ballerina.lib" artifactId = "avro-native" -version = "1.1.1" -path = "../native/build/libs/avro-native-1.1.1-SNAPSHOT.jar" +version = "1.1.0" +path = "../native/build/libs/avro-native-1.1.0-SNAPSHOT.jar" [[platform.java17.dependency]] groupId = "org.apache.avro" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 06bc9d9..e4fdba7 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -10,7 +10,7 @@ distribution-version = "2201.9.0" [[package]] org = "ballerina" name = "avro" -version = "1.1.1" +version = "1.1.0" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"}, From c4a00325059bcda90278c121fe4c771684327a5d Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 9 Oct 2024 15:45:12 +0530 Subject: [PATCH 10/35] Bump the version to 1.1.0 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 4791a28..b28f9fb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.caching=true group=io.ballerina.lib -version=1.1.1-SNAPSHOT +version=1.1.0-SNAPSHOT ballerinaLangVersion=2201.9.0 checkstylePluginVersion=10.12.0 From aff0f129aca1e32905e43a818c362e7021f78bd1 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 9 Oct 2024 16:07:52 +0530 Subject: [PATCH 11/35] Bump the version to 1.0.1 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index b28f9fb..b45ad16 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.caching=true group=io.ballerina.lib -version=1.1.0-SNAPSHOT +version=1.0.1-SNAPSHOT ballerinaLangVersion=2201.9.0 checkstylePluginVersion=10.12.0 From 8bbb6cd40d57d13f9145416ac7674d2406748135 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 9 Oct 2024 16:08:49 +0530 Subject: [PATCH 12/35] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 6 +++--- ballerina/Dependencies.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 2ab57cd..b3e1fed 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "1.1.0" +version = "1.0.1" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] @@ -18,8 +18,8 @@ graalvmCompatible = true [[platform.java17.dependency]] groupId = "io.ballerina.lib" artifactId = "avro-native" -version = "1.1.0" -path = "../native/build/libs/avro-native-1.1.0-SNAPSHOT.jar" +version = "1.0.1" +path = "../native/build/libs/avro-native-1.0.1-SNAPSHOT.jar" [[platform.java17.dependency]] groupId = "org.apache.avro" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index e4fdba7..784c703 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -10,7 +10,7 @@ distribution-version = "2201.9.0" [[package]] org = "ballerina" name = "avro" -version = "1.1.0" +version = "1.0.1" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"}, From defd3727e17e234d205d874b0ec79ec679082802 Mon Sep 17 00:00:00 2001 From: ballerina-bot Date: Wed, 9 Oct 2024 15:20:13 +0000 Subject: [PATCH 13/35] [Automated] Update the native jar versions --- ballerina/Dependencies.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 784c703..64fe075 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -23,7 +23,7 @@ modules = [ [[package]] org = "ballerina" name = "io" -version = "1.6.1" +version = "1.6.0" scope = "testOnly" dependencies = [ {org = "ballerina", name = "jballerina.java"}, From 812e98e7c1be1e7ee04fac976416e155d1ccaded Mon Sep 17 00:00:00 2001 From: ballerina-bot Date: Wed, 9 Oct 2024 15:20:58 +0000 Subject: [PATCH 14/35] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index b3e1fed..80d8934 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -19,7 +19,7 @@ graalvmCompatible = true groupId = "io.ballerina.lib" artifactId = "avro-native" version = "1.0.1" -path = "../native/build/libs/avro-native-1.0.1-SNAPSHOT.jar" +path = "../native/build/libs/avro-native-1.0.1.jar" [[platform.java17.dependency]] groupId = "org.apache.avro" From 92b192a5affd767dc0405ce96e38f0fc39a8de44 Mon Sep 17 00:00:00 2001 From: ballerina-bot Date: Wed, 9 Oct 2024 15:21:04 +0000 Subject: [PATCH 15/35] [Gradle Release Plugin] - pre tag commit: 'v1.0.1'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index b45ad16..7ae9fc3 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.caching=true group=io.ballerina.lib -version=1.0.1-SNAPSHOT +version=1.0.1 ballerinaLangVersion=2201.9.0 checkstylePluginVersion=10.12.0 From 6e642fa5737b164b6922466d4e28459ed503c600 Mon Sep 17 00:00:00 2001 From: ballerina-bot Date: Wed, 9 Oct 2024 15:21:06 +0000 Subject: [PATCH 16/35] [Gradle Release Plugin] - new version commit: 'v1.0.2-SNAPSHOT'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 7ae9fc3..1ac0d4c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.caching=true group=io.ballerina.lib -version=1.0.1 +version=1.0.2-SNAPSHOT ballerinaLangVersion=2201.9.0 checkstylePluginVersion=10.12.0 From 8438b6e308ffd8cce2e010e27f70acc08490dada Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Tue, 15 Oct 2024 13:12:02 +0530 Subject: [PATCH 17/35] Fix generating values for union types --- .../serialize/visitor/SerializeVisitor.java | 127 ++++++------------ 1 file changed, 41 insertions(+), 86 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java index 4c52977..1e011a4 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java @@ -22,6 +22,7 @@ import io.ballerina.lib.avro.serialize.EnumSerializer; import io.ballerina.lib.avro.serialize.FixedSerializer; import io.ballerina.lib.avro.serialize.MapSerializer; +import io.ballerina.lib.avro.serialize.MessageFactory; import io.ballerina.lib.avro.serialize.PrimitiveSerializer; import io.ballerina.lib.avro.serialize.RecordSerializer; import io.ballerina.lib.avro.serialize.Serializer; @@ -39,6 +40,7 @@ import org.apache.avro.generic.GenericRecord; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -102,6 +104,13 @@ public Object visit(PrimitiveSerializer primitiveSerializer, Object data) throws case FLOAT -> { return ((Double) data).floatValue(); } + case DOUBLE -> { + if (data instanceof Long) { + return ((Long) data).doubleValue(); + } else { + return data; + } + } case BYTES -> { ByteBuffer byteBuffer = ByteBuffer.allocate(((BArray) data).getByteArray().length); byteBuffer.put(((BArray) data).getByteArray()); @@ -156,96 +165,42 @@ public GenericData.Array visit(ArraySerializer arraySerializer, BArray d return Objects.requireNonNull(visitor).visit(data, arraySerializer.getSchema(), array); } - public Object visit(UnionSerializer unionSerializer, Object data) throws Exception { - Schema fieldSchema = unionSerializer.getSchema(); - Type typeName = TypeUtils.getType(data); - switch (typeName.getTag()) { - case TypeTags.STRING_TAG -> { - return visitUnionStrings(data, fieldSchema); - } - case TypeTags.ARRAY_TAG -> { - return visitUnionArrays(data, fieldSchema); - } - case TypeTags.MAP_TAG -> { - return new MapSerializer(fieldSchema).convert(this, data); - } - case TypeTags.RECORD_TYPE_TAG -> { - Schema schema = getRecordSchema(Schema.Type.RECORD, fieldSchema.getTypes()); - return new RecordSerializer(schema).convert(this, data); - } - case TypeTags.INT_TAG -> { - return visitUnionIntegers(data, fieldSchema); - } - case TypeTags.FLOAT_TAG -> { - return visitUnionFloats(data, fieldSchema); - } - default -> { - return data; - } - } - } - - private Object visitUnionFloats(Object data, Schema fieldSchema) { - return fieldSchema.getTypes().stream() - .filter(schema -> schema.getType().equals(Schema.Type.FLOAT)) - .findFirst() - .map(schema -> { - try { - return new PrimitiveSerializer(schema).convert(this, data); - } catch (Exception e) { - throw new RuntimeException(e); - } - }) - .orElse(data); - } - - private Object visitUnionIntegers(Object data, Schema fieldSchema) { - return fieldSchema.getTypes().stream() - .filter(schema -> schema.getType().equals(Schema.Type.INT)) - .findFirst() - .map(schema -> { - try { - return new PrimitiveSerializer(schema).convert(this, data); - } catch (Exception e) { - throw new RuntimeException(e); - } - }) - .orElse(data); - } - - private Object visitUnionStrings(Object data, Schema fieldSchema) throws Exception { - return fieldSchema.getTypes().stream() - .filter(type -> type.getType().equals(Schema.Type.ENUM)) - .findFirst() - .map(type -> visit(new EnumSerializer(type), data)) - .orElse(visit(new PrimitiveSerializer(fieldSchema), data.toString())); - } - - private Object visitUnionArrays(Object data, Schema fieldSchema) throws Exception { - for (Schema schema : fieldSchema.getTypes()) { - switch (schema.getType()) { - case BYTES -> { - return new PrimitiveSerializer(schema).convert(this, data); - } - case FIXED -> { - return new FixedSerializer(schema).convert(this, data); - } - case ARRAY -> { - return new ArraySerializer(schema).convert(this, data); - } - } + public ArrayList deriveBallerinaTag(Schema schema) { + ArrayList tags = new ArrayList<>(); + switch (schema.getType()) { + case STRING, ENUM -> tags.add(TypeTags.STRING_TAG); + case FLOAT, DOUBLE -> { + tags.add(TypeTags.FLOAT_TAG); + tags.add(TypeTags.DECIMAL_TAG); + tags.add(TypeTags.INT_TAG); + } + case LONG, INT -> tags.add(TypeTags.INT_TAG); + case BOOLEAN -> tags.add(TypeTags.BOOLEAN_TAG); + case NULL -> tags.add(TypeTags.NULL_TAG); + case RECORD -> tags.add(TypeTags.RECORD_TYPE_TAG); + case ARRAY -> tags.add(TypeTags.ARRAY_TAG); + case MAP -> tags.add(TypeTags.MAP_TAG); + case BYTES, FIXED -> { + tags.add(TypeTags.BYTE_TAG); + tags.add(TypeTags.BYTE_ARRAY_TAG); + tags.add(TypeTags.ARRAY_TAG); + } + default -> tags.add(TypeTags.ANYDATA_TAG); } - return new ArraySerializer(fieldSchema).convert(this, data); + return tags; } - public static Schema getRecordSchema(Schema.Type givenType, List schemas) { - for (Schema schema: schemas) { - if (schema.getType().equals(Schema.Type.UNION)) { - getRecordSchema(givenType, schema.getTypes()); - } else if (schema.getType().equals(givenType)) { - return schema; + public Object visit(UnionSerializer unionSerializer, Object data) throws Exception { + Schema fieldSchema = unionSerializer.getSchema(); + Type typeName = TypeUtils.getType(data); + List types = fieldSchema.getTypes(); + for (Schema type : types) { + ArrayList tags = deriveBallerinaTag(type); + if (tags.contains(typeName.getTag())) { + Serializer serializer = MessageFactory.createMessage(type); + return Objects.requireNonNull(serializer).convert(this, data); } } - return null; + throw new Exception("Value does not match with the Avro union types"); } } From 6aeac9e41ced5ea2b8394bec31ab1c688ab9adc8 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Tue, 15 Oct 2024 13:19:07 +0530 Subject: [PATCH 18/35] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 6 +++--- ballerina/Dependencies.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 80d8934..b5347f1 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "1.0.1" +version = "1.0.2" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] @@ -18,8 +18,8 @@ graalvmCompatible = true [[platform.java17.dependency]] groupId = "io.ballerina.lib" artifactId = "avro-native" -version = "1.0.1" -path = "../native/build/libs/avro-native-1.0.1.jar" +version = "1.0.2" +path = "../native/build/libs/avro-native-1.0.2-SNAPSHOT.jar" [[platform.java17.dependency]] groupId = "org.apache.avro" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 64fe075..398d69e 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -10,7 +10,7 @@ distribution-version = "2201.9.0" [[package]] org = "ballerina" name = "avro" -version = "1.0.1" +version = "1.0.2" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"}, @@ -23,7 +23,7 @@ modules = [ [[package]] org = "ballerina" name = "io" -version = "1.6.0" +version = "1.6.1" scope = "testOnly" dependencies = [ {org = "ballerina", name = "jballerina.java"}, From ceab34cb952d235daf1515dfbf21a056c3703621 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Tue, 15 Oct 2024 15:34:18 +0530 Subject: [PATCH 19/35] Improve visit API for primitive serializer --- .../serialize/visitor/SerializeVisitor.java | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java index 1e011a4..a5b71e8 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java @@ -34,6 +34,7 @@ import io.ballerina.runtime.api.utils.StringUtils; import io.ballerina.runtime.api.utils.TypeUtils; import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BDecimal; import io.ballerina.runtime.api.values.BMap; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; @@ -97,39 +98,32 @@ private Object serializeField(Schema schema, Object fieldData) throws Exception @Override public Object visit(PrimitiveSerializer primitiveSerializer, Object data) throws Exception { - switch (primitiveSerializer.getSchema().getType()) { - case INT -> { - return ((Long) data).intValue(); - } - case FLOAT -> { - return ((Double) data).floatValue(); - } + return switch (primitiveSerializer.getSchema().getType()) { + case INT -> ((Long) data).intValue(); + case FLOAT -> ((Double) data).floatValue(); case DOUBLE -> { - if (data instanceof Long) { - return ((Long) data).doubleValue(); - } else { - return data; + if (data instanceof Long longValue) { + yield longValue.doubleValue(); + } else if (data instanceof BDecimal decimalValue) { + yield decimalValue.floatValue(); } + yield data; } case BYTES -> { ByteBuffer byteBuffer = ByteBuffer.allocate(((BArray) data).getByteArray().length); byteBuffer.put(((BArray) data).getByteArray()); byteBuffer.position(0); - return byteBuffer; - } - case STRING -> { - return data.toString(); + yield byteBuffer; } + case STRING -> data.toString(); case NULL -> { if (data != null) { throw new Exception("The value does not match with the null schema"); } - return null; + yield null; } - default -> { - return data; - } - } + default -> data; + }; } public Map visit(MapSerializer mapSerializer, BMap data) throws Exception { From ebdff39305b4953580baae229ce6b7f11c5bf0ae Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Tue, 15 Oct 2024 15:36:50 +0530 Subject: [PATCH 20/35] Add a test case to validate type casting --- ballerina/tests/record_tests.bal | 102 +++++++++ ballerina/tests/resources/schema_complex.json | 196 ++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 ballerina/tests/resources/schema_complex.json diff --git a/ballerina/tests/record_tests.bal b/ballerina/tests/record_tests.bal index bc26942..3073caf 100644 --- a/ballerina/tests/record_tests.bal +++ b/ballerina/tests/record_tests.bal @@ -370,3 +370,105 @@ public isolated function testOptionalMultipleFieldsInRecords() returns error? { }; return verifyOperation(Lecturer6, lecturer6, schema); } + + +@test:Config { + groups: ["map", "record"] +} +public isolated function testTypeCastingInRecords() returns error? { + string jsonFileName = string `tests/resources/schema_complex.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + Record recordValue = { + hrdCasinoGgrLt: 4999.01, + hrdAccountCreationTimestamp: "2024-02-25T20:50:37.891782", + hrdSportsBetCountLifetime: 77, + hrdSportsFreeBetAmountLifetime: 0, + hriUnityId: "807612199", + hrdLastRealMoneySportsbookBetTs: "2024-04-24T20:15:51.932", + hrdLoyaltyTier: "MEMBER_TIER", + rowInsertTimestampEst: "2024-09-23T03:29:43.431", + ltvSports365Total: 0, + hrdFirstDepositTimestamp: "2024-02-25T20:56:24.109021", + hrdVipStatus: true, + hrdLastRealMoneyCasinoWagerTs: null, + hrdSportsCashBetAmountLifetime: 731, + hrdAccountStatus: "ACTIVE", + hrdAccountId: "1362347172559585332", + ltvAllVerticals365Total: 0, + hrdOptInSms: false, + ltvCasino365Total: null, + hrdAccountSubStatus: null, + hrdCasinoTotalWagerLifetime: null, + hrdSportsGgrLt: 4999.01, + currentGeoSegment: "HRD_FLORIDA", + kycStatus: "VERIFIED", + signupGeoSegment: "HRD_FLORIDA", + hrdSportsBetAmountLifetime: 731, + hrdOptInEmail: false, + hrdOptInPush: false + }; + Schema avro = check new (schema); + byte[] serializedValue = check avro.toAvro(recordValue); + var deserializedValue = check avro.fromAvro(serializedValue, Record); + + json expected = { + "hrdCasinoGgrLt":4999.01, + "hrdAccountCreationTimestamp":"2024-02-25T20:50:37.891782", + "hrdSportsFreeBetAmountLifetime":0.0, + "hriUnityId":"807612199", + "hrdLastRealMoneySportsbookBetTs":"2024-04-24T20:15:51.932", + "hrdLoyaltyTier":"MEMBER_TIER", + "rowInsertTimestampEst":"2024-09-23T03:29:43.431", + "ltvSports365Total":0.0, + "hrdVipStatus":true, + "hrdSportsBetCountLifetime":77.0, + "hrdLastRealMoneyCasinoWagerTs":null, + "hrdSportsCashBetAmountLifetime":731.0, + "hrdAccountStatus":"ACTIVE", + "hrdAccountId":"1362347172559585332", + "ltvAllVerticals365Total":0.0, + "ltvCasino365Total":null, + "hrdAccountSubStatus":null, + "hrdCasinoTotalWagerLifetime":null, + "hrdSportsGgrLt":4999.01, + "currentGeoSegment":"HRD_FLORIDA", + "kycStatus":"VERIFIED", + "hrdOptInSms":false, + "hrdFirstDepositTimestamp":"2024-02-25T20:56:24.109021", + "signupGeoSegment":"HRD_FLORIDA", + "hrdSportsBetAmountLifetime":731.0, + "hrdOptInEmail":false, + "hrdOptInPush":false + }; + test:assertEquals(deserializedValue.toJson(), expected.toJson()); +} + +type Record record { + decimal? hrdCasinoGgrLt; + string? hrdAccountCreationTimestamp; + float? hrdSportsBetCountLifetime; + float? hrdSportsFreeBetAmountLifetime; + string? hriUnityId; + string? hrdLastRealMoneySportsbookBetTs; + string? hrdLoyaltyTier; + string? rowInsertTimestampEst; + float? ltvSports365Total; + string? hrdFirstDepositTimestamp; + boolean? hrdVipStatus; + string? hrdLastRealMoneyCasinoWagerTs; + float? hrdSportsCashBetAmountLifetime; + string? hrdAccountStatus; + string? hrdAccountId; + float? ltvAllVerticals365Total; + boolean? hrdOptInSms; + float? ltvCasino365Total; + string? hrdAccountSubStatus; + float? hrdCasinoTotalWagerLifetime; + float? hrdSportsGgrLt; + string? currentGeoSegment; + string? kycStatus; + string? signupGeoSegment; + float? hrdSportsBetAmountLifetime; + boolean? hrdOptInEmail; + boolean? hrdOptInPush; +}; diff --git a/ballerina/tests/resources/schema_complex.json b/ballerina/tests/resources/schema_complex.json new file mode 100644 index 0000000..145f401 --- /dev/null +++ b/ballerina/tests/resources/schema_complex.json @@ -0,0 +1,196 @@ +{ + "connect.name": "io.confluent.ksql.avro_schemas.KsqlDataSourceSchema", + "fields": [ + { + "name": "hrdCasinoGgrLt", + "type": [ + "null", + "double" + ] + }, + { + "name": "hrdAccountCreationTimestamp", + "type": [ + "null", + "string" + ] + }, + { + "name": "hrdSportsBetCountLifetime", + "type": [ + "null", + "double" + ] + }, + { + "name": "hrdSportsFreeBetAmountLifetime", + "type": [ + "null", + "double" + ] + }, + { + "name": "hriUnityId", + "type": [ + "null", + "string" + ] + }, + { + "name": "hrdLastRealMoneySportsbookBetTs", + "type": [ + "null", + "string" + ] + }, + { + "name": "hrdLoyaltyTier", + "type": [ + "null", + "string" + ] + }, + { + "name": "rowInsertTimestampEst", + "type": [ + "null", + "string" + ] + }, + { + "name": "ltvSports365Total", + "type": [ + "null", + "double" + ] + }, + { + "name": "hrdFirstDepositTimestamp", + "type": [ + "null", + "string" + ] + }, + { + "name": "hrdVipStatus", + "type": [ + "null", + "boolean" + ] + }, + { + "name": "hrdLastRealMoneyCasinoWagerTs", + "type": [ + "null", + "string" + ] + }, + { + "name": "hrdSportsCashBetAmountLifetime", + "type": [ + "null", + "double" + ] + }, + { + "name": "hrdAccountStatus", + "type": [ + "null", + "string" + ] + }, + { + "name": "hrdAccountId", + "type": [ + "null", + "string" + ] + }, + { + "name": "ltvAllVerticals365Total", + "type": [ + "null", + "double" + ] + }, + { + "name": "hrdOptInSms", + "type": [ + "null", + "boolean" + ] + }, + { + "name": "ltvCasino365Total", + "type": [ + "null", + "double" + ] + }, + { + "name": "hrdAccountSubStatus", + "type": [ + "null", + "string" + ] + }, + { + "name": "hrdCasinoTotalWagerLifetime", + "type": [ + "null", + "double" + ] + }, + { + "name": "hrdSportsGgrLt", + "type": [ + "null", + "double" + ] + }, + { + "name": "currentGeoSegment", + "type": [ + "null", + "string" + ] + }, + { + "name": "kycStatus", + "type": [ + "null", + "string" + ] + }, + { + "name": "signupGeoSegment", + "type": [ + "null", + "string" + ] + }, + { + "name": "hrdSportsBetAmountLifetime", + "type": [ + "null", + "double" + ] + }, + { + "name": "hrdOptInEmail", + "type": [ + "null", + "boolean" + ] + }, + { + "name": "hrdOptInPush", + "type": [ + "null", + "boolean" + ] + } + ], + "name": "Record", + "type": "record" +} From 12493295e3216953434042340e9d7aef10858fb0 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Tue, 15 Oct 2024 16:30:59 +0530 Subject: [PATCH 21/35] Fix test group for a test case --- ballerina/tests/record_tests.bal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/tests/record_tests.bal b/ballerina/tests/record_tests.bal index 3073caf..63bb726 100644 --- a/ballerina/tests/record_tests.bal +++ b/ballerina/tests/record_tests.bal @@ -373,7 +373,7 @@ public isolated function testOptionalMultipleFieldsInRecords() returns error? { @test:Config { - groups: ["map", "record"] + groups: ["record", "union"] } public isolated function testTypeCastingInRecords() returns error? { string jsonFileName = string `tests/resources/schema_complex.json`; From 3e40148cd67ed8052cc71a3374bc200de4a7ad10 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 16 Oct 2024 02:25:39 +0530 Subject: [PATCH 22/35] Apply review suggestions --- .../avro/serialize/visitor/SerializeVisitor.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java index a5b71e8..527655e 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java @@ -99,8 +99,18 @@ private Object serializeField(Schema schema, Object fieldData) throws Exception @Override public Object visit(PrimitiveSerializer primitiveSerializer, Object data) throws Exception { return switch (primitiveSerializer.getSchema().getType()) { - case INT -> ((Long) data).intValue(); - case FLOAT -> ((Double) data).floatValue(); + case INT -> { + if (data instanceof Long longValue) { + yield longValue.intValue(); + } + yield data; + } + case FLOAT -> { + if (data instanceof Double doubleValue) { + yield doubleValue.floatValue(); + } + yield data; + } case DOUBLE -> { if (data instanceof Long longValue) { yield longValue.doubleValue(); From a05d61bfa3e4bb399001436e5df0cefe9bf2b20b Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 16 Oct 2024 02:26:10 +0530 Subject: [PATCH 23/35] Add test cases to validate short and byte values --- ballerina/tests/primitive_tests.bal | 64 +++++++++++++++++++++++++++++ ballerina/tests/types.bal | 31 +++++++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/ballerina/tests/primitive_tests.bal b/ballerina/tests/primitive_tests.bal index 0b7add8..5d3ebaa 100644 --- a/ballerina/tests/primitive_tests.bal +++ b/ballerina/tests/primitive_tests.bal @@ -134,3 +134,67 @@ public isolated function testNullValuesWithNonNullData() returns error? { byte[]|error serializedValue = avro.toAvro("string"); test:assertTrue(serializedValue is error); } + +@test:Config { + groups: ["primitive"] +} +public isolated function testUnsignedShortValue() returns error? { + string schema = string ` + { + "type": "int", + "name" : "shortValue", + "namespace": "data" + }`; + + int:Unsigned8 value = 255; + return verifyOperation(int:Unsigned8, value, schema); +} + +@test:Config { + groups: ["primitive", "check"] +} +public isolated function testSignedShortValue() returns error? { + string schema = string ` + { + "type": "int", + "name" : "shortValue", + "namespace": "data" + }`; + + int:Signed32 value = 555950000; + return verifyOperation(int:Signed32, value, schema); +} + +@test:Config { + groups: ["primitive", "check"] +} +public isolated function testSignedMinusShortValue() returns error? { + string schema = string ` + { + "type": "double", + "name" : "shortValue", + "namespace": "data" + }`; + + int:Signed32 value = -2147483; + Schema avro = check new (schema); + byte[] serializedValue = check avro.toAvro(value); + float deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, -2147483.0); +} + +@test:Config { + groups: ["primitive", "check"] +} +public isolated function testByteValue() returns error? { + string schema = string ` + { + "type": "bytes", + "name" : "byteValue", + "namespace": "data" + }`; + byte[] data = []; + byte value = 2; + data.push(value); + return verifyOperation(ByteArray, data, schema); +} diff --git a/ballerina/tests/types.bal b/ballerina/tests/types.bal index 0c613c9..542dc61 100644 --- a/ballerina/tests/types.bal +++ b/ballerina/tests/types.bal @@ -271,7 +271,6 @@ public type Envelope2 record { string? MessageSource; }; - public type UnionEnumRecord record { string|Numbers? field1; }; @@ -288,6 +287,36 @@ public type ReadOnlyRec readonly & record { string|UnionEnumRecord? & readonly field1; }; +type Record record { + decimal? hrdCasinoGgrLt; + string? hrdAccountCreationTimestamp; + float? hrdSportsBetCountLifetime; + float? hrdSportsFreeBetAmountLifetime; + string? hriUnityId; + string? hrdLastRealMoneySportsbookBetTs; + string? hrdLoyaltyTier; + string? rowInsertTimestampEst; + float? ltvSports365Total; + string? hrdFirstDepositTimestamp; + boolean? hrdVipStatus; + string? hrdLastRealMoneyCasinoWagerTs; + float? hrdSportsCashBetAmountLifetime; + string? hrdAccountStatus; + string? hrdAccountId; + float? ltvAllVerticals365Total; + boolean? hrdOptInSms; + float? ltvCasino365Total; + string? hrdAccountSubStatus; + float? hrdCasinoTotalWagerLifetime; + float? hrdSportsGgrLt; + string? currentGeoSegment; + string? kycStatus; + string? signupGeoSegment; + float? hrdSportsBetAmountLifetime; + boolean? hrdOptInEmail; + boolean? hrdOptInPush; +}; + type ReadOnlyUnionFixed UnionFixedRecord & readonly; type ByteArray byte[]; type ReadOnlyIntArray int[] & readonly; From c4b6014719efe249ef0abaa887c9d257180879af Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Wed, 16 Oct 2024 02:28:52 +0530 Subject: [PATCH 24/35] Update the code to map the values to the first suitable type in union types --- ballerina/tests/record_tests.bal | 56 +++++++++---------- ballerina/tests/types.bal | 1 + .../deserialize/visitor/UnionRecordUtils.java | 43 +++++--------- 3 files changed, 40 insertions(+), 60 deletions(-) diff --git a/ballerina/tests/record_tests.bal b/ballerina/tests/record_tests.bal index 63bb726..2335abc 100644 --- a/ballerina/tests/record_tests.bal +++ b/ballerina/tests/record_tests.bal @@ -371,7 +371,6 @@ public isolated function testOptionalMultipleFieldsInRecords() returns error? { return verifyOperation(Lecturer6, lecturer6, schema); } - @test:Config { groups: ["record", "union"] } @@ -409,7 +408,7 @@ public isolated function testTypeCastingInRecords() returns error? { }; Schema avro = check new (schema); byte[] serializedValue = check avro.toAvro(recordValue); - var deserializedValue = check avro.fromAvro(serializedValue, Record); + Record deserializedValue = check avro.fromAvro(serializedValue); json expected = { "hrdCasinoGgrLt":4999.01, @@ -443,32 +442,27 @@ public isolated function testTypeCastingInRecords() returns error? { test:assertEquals(deserializedValue.toJson(), expected.toJson()); } -type Record record { - decimal? hrdCasinoGgrLt; - string? hrdAccountCreationTimestamp; - float? hrdSportsBetCountLifetime; - float? hrdSportsFreeBetAmountLifetime; - string? hriUnityId; - string? hrdLastRealMoneySportsbookBetTs; - string? hrdLoyaltyTier; - string? rowInsertTimestampEst; - float? ltvSports365Total; - string? hrdFirstDepositTimestamp; - boolean? hrdVipStatus; - string? hrdLastRealMoneyCasinoWagerTs; - float? hrdSportsCashBetAmountLifetime; - string? hrdAccountStatus; - string? hrdAccountId; - float? ltvAllVerticals365Total; - boolean? hrdOptInSms; - float? ltvCasino365Total; - string? hrdAccountSubStatus; - float? hrdCasinoTotalWagerLifetime; - float? hrdSportsGgrLt; - string? currentGeoSegment; - string? kycStatus; - string? signupGeoSegment; - float? hrdSportsBetAmountLifetime; - boolean? hrdOptInEmail; - boolean? hrdOptInPush; -}; +@test:Config { + groups: ["record", "union"] +} +public isolated function testUnionsWithRecords() returns error? { + string schema = string `{ + "type": "record", + "name": "DataRecord", + "namespace": "data", + "fields": [ + { + "name": "shortValue", + "type": ["null", "int", "double"] + } + ] + }`; + int:Signed16 short = 5555; + json value = { + "shortValue": short + }; + Schema avro = check new (schema); + byte[] serializedValue = check avro.toAvro(value); + DataRecord deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, value); +} diff --git a/ballerina/tests/types.bal b/ballerina/tests/types.bal index 542dc61..f68869b 100644 --- a/ballerina/tests/types.bal +++ b/ballerina/tests/types.bal @@ -363,3 +363,4 @@ type FloatArray float[]; type EnumArray Numbers[]; type Enum2DArray Numbers[][]; type ReadOnlyString2DArray string[][] & readonly; +type DataRecord record{}; diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java index b56484f..c631945 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java @@ -39,41 +39,26 @@ public class UnionRecordUtils { public static void visitUnionRecords(Type type, BMap ballerinaRecord, Schema.Field field, Object fieldData) throws Exception { + int size = ballerinaRecord.size(); for (Schema schemaType : field.schema().getTypes()) { if (fieldData == null) { ballerinaRecord.put(StringUtils.fromString(field.name()), null); break; } switch (schemaType.getType()) { - case BYTES: - handleBytesField(field, fieldData, ballerinaRecord); - break; - case FIXED: - handleFixedField(field, fieldData, ballerinaRecord); - break; - case ARRAY: - handleArrayField(field, fieldData, ballerinaRecord, schemaType); - break; - case MAP: - handleMapField(field, fieldData, ballerinaRecord); - break; - case RECORD: - handleRecordField(type, field, fieldData, ballerinaRecord, schemaType); - break; - case STRING: - handleStringField(field, fieldData, ballerinaRecord); - break; - case INT, LONG: - handleIntegerField(field, fieldData, ballerinaRecord); - break; - case FLOAT, DOUBLE: - handleFloatField(field, fieldData, ballerinaRecord); - break; - case ENUM: - handleEnumField(field, fieldData, ballerinaRecord); - break; - default: - handleDefaultField(field, fieldData, ballerinaRecord); + case BYTES -> handleBytesField(field, fieldData, ballerinaRecord); + case FIXED -> handleFixedField(field, fieldData, ballerinaRecord); + case ARRAY -> handleArrayField(field, fieldData, ballerinaRecord, schemaType); + case MAP -> handleMapField(field, fieldData, ballerinaRecord); + case RECORD -> handleRecordField(type, field, fieldData, ballerinaRecord, schemaType); + case STRING -> handleStringField(field, fieldData, ballerinaRecord); + case INT, LONG -> handleIntegerField(field, fieldData, ballerinaRecord); + case FLOAT, DOUBLE -> handleFloatField(field, fieldData, ballerinaRecord); + case ENUM -> handleEnumField(field, fieldData, ballerinaRecord); + default -> handleDefaultField(field, fieldData, ballerinaRecord); + } + if (ballerinaRecord.size() != size) { + break; } } } From 071f0073b9caa11c256614b2aafddc7f9d1e4a1a Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Fri, 18 Oct 2024 10:01:24 +0530 Subject: [PATCH 25/35] Update changelog for 1.0.1 and 1.0.2 releases --- changelog.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index cdf13c9..46c46a8 100644 --- a/changelog.md +++ b/changelog.md @@ -5,8 +5,20 @@ This file contains all the notable changes done to the Ballerina Avro package th The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.0.2] - 2024-10-18 + +### Fixed + +- [[#27] Fix handling union type values in the Avro payload](https://github.com/ballerina-platform/module-ballerina-avro/pull/27) + +## [1.0.1] - 2024-10-10 + +### Fixed + +- [[#25] Fix `CVE-2024-47561` security vulnerability](https://github.com/ballerina-platform/module-ballerina-avro/pull/25) + ## [1.0.0] - 2024-05-13 ### Added -- [[#6463] Introduce `byte` support to Avro module](https://github.com/ballerina-platform/ballerina-library/issues/6463) +- [[#17] Introduce `byte` support to Avro module](https://github.com/ballerina-platform/ballerina-library/issues/6463) From ff913717b1e262129bb77d372f2fcb95a2a6e5f7 Mon Sep 17 00:00:00 2001 From: ballerina-bot Date: Fri, 18 Oct 2024 05:26:16 +0000 Subject: [PATCH 26/35] [Automated] Update the native jar versions --- ballerina/Dependencies.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 398d69e..84018b8 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -23,7 +23,7 @@ modules = [ [[package]] org = "ballerina" name = "io" -version = "1.6.1" +version = "1.6.0" scope = "testOnly" dependencies = [ {org = "ballerina", name = "jballerina.java"}, From 80fb9309b5ce6d544afb1dd859e089232d3cde8d Mon Sep 17 00:00:00 2001 From: ballerina-bot Date: Fri, 18 Oct 2024 05:27:00 +0000 Subject: [PATCH 27/35] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index b5347f1..0c3227d 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -19,7 +19,7 @@ graalvmCompatible = true groupId = "io.ballerina.lib" artifactId = "avro-native" version = "1.0.2" -path = "../native/build/libs/avro-native-1.0.2-SNAPSHOT.jar" +path = "../native/build/libs/avro-native-1.0.2.jar" [[platform.java17.dependency]] groupId = "org.apache.avro" From 762a2c3b7bb1e9d5e95da8b81dfbb15ff08d92d7 Mon Sep 17 00:00:00 2001 From: ballerina-bot Date: Fri, 18 Oct 2024 05:27:06 +0000 Subject: [PATCH 28/35] [Gradle Release Plugin] - pre tag commit: 'v1.0.2'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 1ac0d4c..b361cd5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.caching=true group=io.ballerina.lib -version=1.0.2-SNAPSHOT +version=1.0.2 ballerinaLangVersion=2201.9.0 checkstylePluginVersion=10.12.0 From 642b9e27162d69062890e3325d262ea5e4c2ac41 Mon Sep 17 00:00:00 2001 From: ballerina-bot Date: Fri, 18 Oct 2024 05:27:08 +0000 Subject: [PATCH 29/35] [Gradle Release Plugin] - new version commit: 'v1.0.3-SNAPSHOT'. --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index b361cd5..bbbf289 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ org.gradle.caching=true group=io.ballerina.lib -version=1.0.2 +version=1.0.3-SNAPSHOT ballerinaLangVersion=2201.9.0 checkstylePluginVersion=10.12.0 From a9fc578680d7a5a474e925ac196b5d4d2314f0e4 Mon Sep 17 00:00:00 2001 From: hindujaB Date: Tue, 12 Nov 2024 17:12:40 +0530 Subject: [PATCH 30/35] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 14 +++++++------- ballerina/Dependencies.toml | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 80ba45f..a86c583 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "1.0.0" +version = "1.0.1" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] @@ -12,22 +12,22 @@ distribution = "2201.8.0" [build-options] observabilityIncluded = true -[platform.java17] +[platform.java21] graalvmCompatible = true -[[platform.java17.dependency]] +[[platform.java21.dependency]] groupId = "io.ballerina.lib" artifactId = "avro-native" -version = "1.0.0" -path = "../native/build/libs/avro-native-1.0.0.jar" +version = "1.0.1" +path = "../native/build/libs/avro-native-1.0.1-SNAPSHOT.jar" -[[platform.java17.dependency]] +[[platform.java21.dependency]] groupId = "org.apache.avro" artifactId = "avro" version = "1.11.3" path = "./lib/avro-1.11.3.jar" -[[platform.java17.dependency]] +[[platform.java21.dependency]] groupId = "com.fasterxml.jackson.core" artifactId = "jackson-core" version = "2.17.0" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index e770f53..7214a15 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -5,12 +5,12 @@ [ballerina] dependencies-toml-version = "2" -distribution-version = "2201.9.0" +distribution-version = "2201.10.0-20240926-231800-8a5a4343" [[package]] org = "ballerina" name = "avro" -version = "1.0.0" +version = "1.0.1" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"}, @@ -23,7 +23,7 @@ modules = [ [[package]] org = "ballerina" name = "io" -version = "1.6.0" +version = "1.6.2" scope = "testOnly" dependencies = [ {org = "ballerina", name = "jballerina.java"}, From 3cca49919a1719c64449609ddbd53d33815831ef Mon Sep 17 00:00:00 2001 From: hindujaB Date: Wed, 13 Nov 2024 06:13:23 +0530 Subject: [PATCH 31/35] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 6 +++--- ballerina/Dependencies.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 5b71219..4eaf6cd 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "1.0.2" +version = "1.0.3" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] @@ -18,8 +18,8 @@ graalvmCompatible = true [[platform.java21.dependency]] groupId = "io.ballerina.lib" artifactId = "avro-native" -version = "1.0.2" -path = "../native/build/libs/avro-native-1.0.2.jar" +version = "1.0.3" +path = "../native/build/libs/avro-native-1.0.3-SNAPSHOT.jar" [[platform.java21.dependency]] groupId = "org.apache.avro" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index b486235..fb7972b 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -10,7 +10,7 @@ distribution-version = "2201.10.0-20240926-231800-8a5a4343" [[package]] org = "ballerina" name = "avro" -version = "1.0.2" +version = "1.0.3" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"}, From 9584e350e0004266ed947723301ed06da7b84a51 Mon Sep 17 00:00:00 2001 From: hindujaB Date: Wed, 13 Nov 2024 06:27:04 +0530 Subject: [PATCH 32/35] [Automated] Update the native jar versions --- ballerina/Dependencies.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index fb7972b..0bbd418 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -5,7 +5,7 @@ [ballerina] dependencies-toml-version = "2" -distribution-version = "2201.10.0-20240926-231800-8a5a4343" +distribution-version = "2201.11.0-20241112-214900-6b80ab87" [[package]] org = "ballerina" From 59db76c0ae58e8b487ed9887fafd2326a0a7e699 Mon Sep 17 00:00:00 2001 From: hindujaB Date: Wed, 13 Nov 2024 06:27:28 +0530 Subject: [PATCH 33/35] Update runtime APIs --- gradle.properties | 4 ++-- native/src/main/java/io/ballerina/lib/avro/Utils.java | 2 +- .../lib/avro/deserialize/visitor/DeserializeArrayVisitor.java | 2 +- .../lib/avro/deserialize/visitor/DeserializeVisitor.java | 2 +- .../lib/avro/serialize/visitor/SerializeVisitor.java | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gradle.properties b/gradle.properties index e85c089..70eeb6b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,7 @@ org.gradle.caching=true group=io.ballerina.lib version=1.0.3-SNAPSHOT -ballerinaLangVersion=2201.10.0-20240926-231800-8a5a4343 +ballerinaLangVersion=2201.11.0-20241112-214900-6b80ab87 checkstylePluginVersion=10.12.0 spotbugsPluginVersion=6.0.18 @@ -13,6 +13,6 @@ releasePluginVersion=2.8.0 ballerinaGradlePluginVersion=2.0.1 # Dependencies -stdlibIoVersion=1.6.2-20240928-084100-656404f +stdlibIoVersion=1.6.2-20241112-233100-995cf5f avroVersion=1.11.4 jacksonVersion=2.18.0 diff --git a/native/src/main/java/io/ballerina/lib/avro/Utils.java b/native/src/main/java/io/ballerina/lib/avro/Utils.java index 3f2bd5a..fea5b1d 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Utils.java +++ b/native/src/main/java/io/ballerina/lib/avro/Utils.java @@ -18,10 +18,10 @@ package io.ballerina.lib.avro; -import io.ballerina.runtime.api.TypeTags; import io.ballerina.runtime.api.creators.ErrorCreator; import io.ballerina.runtime.api.types.IntersectionType; import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.types.TypeTags; import io.ballerina.runtime.api.utils.StringUtils; import io.ballerina.runtime.api.utils.TypeUtils; import io.ballerina.runtime.api.values.BError; diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java index 5bb11c2..49d0f86 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java @@ -21,10 +21,10 @@ import io.ballerina.lib.avro.Utils; import io.ballerina.lib.avro.deserialize.ArrayDeserializer; import io.ballerina.lib.avro.deserialize.Deserializer; -import io.ballerina.runtime.api.TypeTags; import io.ballerina.runtime.api.creators.ValueCreator; import io.ballerina.runtime.api.types.ArrayType; import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.types.TypeTags; import io.ballerina.runtime.api.values.BArray; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java index b03d8eb..a9a721a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -27,7 +27,6 @@ import io.ballerina.lib.avro.deserialize.PrimitiveDeserializer; import io.ballerina.lib.avro.deserialize.RecordDeserializer; import io.ballerina.lib.avro.deserialize.UnionDeserializer; -import io.ballerina.runtime.api.TypeTags; import io.ballerina.runtime.api.creators.ValueCreator; import io.ballerina.runtime.api.types.ArrayType; import io.ballerina.runtime.api.types.Field; @@ -35,6 +34,7 @@ import io.ballerina.runtime.api.types.RecordType; import io.ballerina.runtime.api.types.ReferenceType; import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.types.TypeTags; import io.ballerina.runtime.api.utils.StringUtils; import io.ballerina.runtime.api.utils.TypeUtils; import io.ballerina.runtime.api.utils.ValueUtils; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java index 527655e..9e01b0f 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java @@ -29,8 +29,8 @@ import io.ballerina.lib.avro.serialize.UnionSerializer; import io.ballerina.lib.avro.serialize.visitor.array.ArrayVisitorFactory; import io.ballerina.lib.avro.serialize.visitor.array.IArrayVisitor; -import io.ballerina.runtime.api.TypeTags; import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.types.TypeTags; import io.ballerina.runtime.api.utils.StringUtils; import io.ballerina.runtime.api.utils.TypeUtils; import io.ballerina.runtime.api.values.BArray; From e268873b56b3908d57434b7550edf72b441c4596 Mon Sep 17 00:00:00 2001 From: hindujaB Date: Wed, 13 Nov 2024 06:28:09 +0530 Subject: [PATCH 34/35] Revert "[Automated] Update the native jar versions" This reverts commit 9584e350e0004266ed947723301ed06da7b84a51. --- ballerina/Dependencies.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 0bbd418..fb7972b 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -5,7 +5,7 @@ [ballerina] dependencies-toml-version = "2" -distribution-version = "2201.11.0-20241112-214900-6b80ab87" +distribution-version = "2201.10.0-20240926-231800-8a5a4343" [[package]] org = "ballerina" From 8855677f737fc4d96b349cf45fba6558ea992f11 Mon Sep 17 00:00:00 2001 From: hindujaB Date: Wed, 13 Nov 2024 06:28:13 +0530 Subject: [PATCH 35/35] Revert "[Automated] Update the native jar versions" This reverts commit 3cca49919a1719c64449609ddbd53d33815831ef. --- ballerina/Ballerina.toml | 6 +++--- ballerina/Dependencies.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 4eaf6cd..5b71219 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "1.0.3" +version = "1.0.2" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] @@ -18,8 +18,8 @@ graalvmCompatible = true [[platform.java21.dependency]] groupId = "io.ballerina.lib" artifactId = "avro-native" -version = "1.0.3" -path = "../native/build/libs/avro-native-1.0.3-SNAPSHOT.jar" +version = "1.0.2" +path = "../native/build/libs/avro-native-1.0.2.jar" [[platform.java21.dependency]] groupId = "org.apache.avro" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index fb7972b..b486235 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -10,7 +10,7 @@ distribution-version = "2201.10.0-20240926-231800-8a5a4343" [[package]] org = "ballerina" name = "avro" -version = "1.0.3" +version = "1.0.2" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"},