From 431711fe3186f063e4812c1486038cfdb72eabdb Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:32:55 +0530 Subject: [PATCH 01/47] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 12 +++--------- ballerina/Dependencies.toml | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 36cac9b..1d79af3 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "0.1.2" +version = "0.1.3" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] @@ -18,8 +18,8 @@ graalvmCompatible = true [[platform.java17.dependency]] groupId = "io.ballerina.lib" artifactId = "avro-native" -version = "0.1.2" -path = "../native/build/libs/avro-native-0.1.2.jar" +version = "0.1.3" +path = "../native/build/libs/avro-native-0.1.3-SNAPSHOT.jar" [[platform.java17.dependency]] groupId = "org.apache.avro" @@ -44,9 +44,3 @@ groupId = "com.fasterxml.jackson.core" artifactId = "jackson-databind" version = "2.17.0" path = "./lib/jackson-databind-2.17.0.jar" - -[[platform.java11.dependency]] -groupId = "com.fasterxml.jackson.dataformat" -artifactId = "jackson-dataformat-avro" -version = "2.17.0" -path = "./lib/jackson-dataformat-avro-2.17.0.jar" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 98def43..02a1600 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -10,7 +10,7 @@ distribution-version = "2201.9.0" [[package]] org = "ballerina" name = "avro" -version = "0.1.2" +version = "0.1.3" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"}, From 4d4594b351623b7f76ba7ceaea8c2cc369e43565 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:35:18 +0530 Subject: [PATCH 02/47] Remove unnecessary avro jackson dataformat version --- ballerina/build.gradle | 3 --- build-config/resources/Ballerina.toml | 6 ------ gradle.properties | 1 - native/build.gradle | 3 +-- native/src/main/java/module-info.java | 3 +-- 5 files changed, 2 insertions(+), 14 deletions(-) diff --git a/ballerina/build.gradle b/ballerina/build.gradle index 734004c..f0d3337 100644 --- a/ballerina/build.gradle +++ b/ballerina/build.gradle @@ -72,9 +72,6 @@ dependencies { externalJars(group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jacksonVersion}") { transitive = false } - externalJars(group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-avro', version: "${jacksonVersion}") { - transitive = false - } } task updateTomlFiles { diff --git a/build-config/resources/Ballerina.toml b/build-config/resources/Ballerina.toml index 4a41f89..c5e7203 100644 --- a/build-config/resources/Ballerina.toml +++ b/build-config/resources/Ballerina.toml @@ -44,9 +44,3 @@ groupId = "com.fasterxml.jackson.core" artifactId = "jackson-databind" version = "@jackson.version@" path = "./lib/jackson-databind-@jackson.version@.jar" - -[[platform.java11.dependency]] -groupId = "com.fasterxml.jackson.dataformat" -artifactId = "jackson-dataformat-avro" -version = "@jackson.version@" -path = "./lib/jackson-dataformat-avro-@jackson.version@.jar" diff --git a/gradle.properties b/gradle.properties index 49e9e58..81030da 100644 --- a/gradle.properties +++ b/gradle.properties @@ -15,5 +15,4 @@ ballerinaGradlePluginVersion=2.0.1 # Dependencies stdlibIoVersion=1.6.0 avroVersion=1.11.3 -avroJacksonVersion=1.8.2 jacksonVersion=2.17.0 diff --git a/native/build.gradle b/native/build.gradle index b883954..feecbbf 100644 --- a/native/build.gradle +++ b/native/build.gradle @@ -30,8 +30,7 @@ dependencies { implementation group: 'org.ballerinalang', name: 'ballerina-lang', version: "${ballerinaLangVersion}" implementation group: 'org.ballerinalang', name: 'ballerina-runtime', version: "${ballerinaLangVersion}" - implementation group: 'org.apache.avro', name: 'avro', version: "${avroJacksonVersion}" - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-avro', version: "${jacksonVersion}" + implementation group: 'org.apache.avro', name: 'avro', version: "${avroVersion}" } checkstyle { diff --git a/native/src/main/java/module-info.java b/native/src/main/java/module-info.java index 6b198c0..11b1546 100644 --- a/native/src/main/java/module-info.java +++ b/native/src/main/java/module-info.java @@ -20,6 +20,5 @@ requires io.ballerina.runtime; requires io.ballerina.lang; requires com.fasterxml.jackson.databind; - requires com.fasterxml.jackson.dataformat.avro; - requires avro; + requires org.apache.avro; } From 73655c4001de02d5275cbe40ebdfe5d732956c33 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:39:01 +0530 Subject: [PATCH 03/47] Add visitable classes for each Avro schema type --- .../lib/avro/serialize/ArraySerializer.java | 35 ++++++++++++++++ .../lib/avro/serialize/EnumSerializer.java | 34 +++++++++++++++ .../lib/avro/serialize/GenericSerializer.java | 29 +++++++++++++ .../lib/avro/serialize/MapSerializer.java | 35 ++++++++++++++++ .../lib/avro/serialize/NullSerializer.java | 32 +++++++++++++++ .../lib/avro/serialize/RecordSerializer.java | 35 ++++++++++++++++ .../lib/avro/serialize/Serializer.java | 41 +++++++++++++++++++ 7 files changed, 241 insertions(+) create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java new file mode 100644 index 0000000..f117529 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; + +public class ArraySerializer extends Serializer { + + public ArraySerializer(Schema schema) { + super(schema); + } + + @Override + public Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception { + return serializeVisitor.visitArray((BArray) data, getSchema()); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java new file mode 100644 index 0000000..d14bd3c --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.visitor.SerializeVisitor; +import org.apache.avro.Schema; + +public class EnumSerializer extends Serializer { + + public EnumSerializer(Schema schema) { + super(schema); + } + + @Override + public Object generateMessage(SerializeVisitor serializeVisitor, Object data) { + return serializeVisitor.visitEnum(data, getSchema()); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java new file mode 100644 index 0000000..1f2fe04 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.visitor.SerializeVisitor; + +public class GenericSerializer extends Serializer { + + @Override + public Object generateMessage(SerializeVisitor serializeVisitor, Object data) { + return data; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java new file mode 100644 index 0000000..d2da813 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.Schema; + +public class MapSerializer extends Serializer { + + public MapSerializer(Schema schema) { + super(schema); + } + + @Override + public Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception { + return serializeVisitor.visitMap((BMap) data, getSchema()); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java new file mode 100644 index 0000000..d6b2345 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.visitor.SerializeVisitor; + +public class NullSerializer extends Serializer { + + @Override + public Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception { + if (data != null) { + throw new Exception("The value does not match with the null schema"); + } + return null; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java new file mode 100644 index 0000000..63e5825 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.Schema; + +public class RecordSerializer extends Serializer { + + public RecordSerializer(Schema schema) { + super(schema); + } + + @Override + public Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception { + return serializeVisitor.visitRecord((BMap) data, getSchema()); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java new file mode 100644 index 0000000..45a788b --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.visitor.SerializeVisitor; +import org.apache.avro.Schema; + +public abstract class Serializer { + + private final Schema schema; + + public Serializer() { + this.schema = null; + } + + public Serializer(Schema schema) { + this.schema = new Schema.Parser().parse(schema.toString()); + } + + protected Schema getSchema() { + return schema; + } + + public abstract Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception; +} From 9e1251e47c230c6c3af7b2070ca4929df0784443 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:42:30 +0530 Subject: [PATCH 04/47] Add factory pattern to generate visitable Avro schema objects --- .../main/java/io/ballerina/lib/avro/Avro.java | 51 +++++++++++-------- .../lib/avro/serialize/MessageFactory.java | 36 +++++++++++++ 2 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java diff --git a/native/src/main/java/io/ballerina/lib/avro/Avro.java b/native/src/main/java/io/ballerina/lib/avro/Avro.java index 9d067fd..a8d8bd1 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Avro.java +++ b/native/src/main/java/io/ballerina/lib/avro/Avro.java @@ -18,27 +18,34 @@ package io.ballerina.lib.avro; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.avro.AvroMapper; -import com.fasterxml.jackson.dataformat.avro.AvroSchema; +import io.ballerina.lib.avro.deserialize.DeserializeFactory; +import io.ballerina.lib.avro.deserialize.Deserializer; +import io.ballerina.lib.avro.serialize.MessageFactory; +import io.ballerina.lib.avro.serialize.Serializer; +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import io.ballerina.lib.avro.visitor.SerializeVisitor; import io.ballerina.runtime.api.creators.ValueCreator; -import io.ballerina.runtime.api.utils.JsonUtils; -import io.ballerina.runtime.api.utils.StringUtils; -import io.ballerina.runtime.api.utils.ValueUtils; import io.ballerina.runtime.api.values.BArray; import io.ballerina.runtime.api.values.BObject; import io.ballerina.runtime.api.values.BString; import io.ballerina.runtime.api.values.BTypedesc; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; -import java.io.IOException; +import java.io.ByteArrayOutputStream; import java.util.Objects; import static io.ballerina.lib.avro.Utils.AVRO_SCHEMA; import static io.ballerina.lib.avro.Utils.DESERIALIZATION_ERROR; -import static io.ballerina.lib.avro.Utils.JSON_PROCESSING_ERROR; +import static io.ballerina.lib.avro.Utils.SERIALIZATION_ERROR; +import static io.ballerina.lib.avro.Utils.createError; public final class Avro { @@ -52,18 +59,18 @@ public static void generateSchema(BObject schemaObject, BString schema) { public static Object toAvro(BObject schemaObject, Object data) { Schema schema = (Schema) schemaObject.getNativeData(AVRO_SCHEMA); - ObjectMapper objectMapper = new ObjectMapper(); - try { - Object jsonObject = generateJsonObject(data, schema, objectMapper); - if (jsonObject == null) { - return ValueCreator.createArrayValue(new byte[]{0, 0}); - } else if (Objects.equals(schema.getType(), Schema.Type.FIXED)) { - return ValueCreator.createArrayValue(((BArray) jsonObject).getByteArray()); - } - byte[] avroBytes = (new AvroMapper()).writer(new AvroSchema(schema)).writeValueAsBytes(jsonObject); - return ValueCreator.createArrayValue(avroBytes); - } catch (JsonProcessingException e) { - return Utils.createError(JSON_PROCESSING_ERROR, e); + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + SerializeVisitor serializeVisitor = new SerializeVisitor(); + Serializer serializer = MessageFactory.createMessage(schema); + Object avroData = Objects.requireNonNull(serializer).generateMessage(serializeVisitor, data); + DatumWriter writer = new GenericDatumWriter<>(schema); + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null); + writer.write(avroData, encoder); + encoder.flush(); + byte[] bytes = outputStream.toByteArray(); + return ValueCreator.createArrayValue(bytes); + } catch (Exception e) { + return Utils.createError(SERIALIZATION_ERROR, e); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java new file mode 100644 index 0000000..150b1b4 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import org.apache.avro.Schema; + +public class MessageFactory { + + public static Serializer createMessage(Schema schema) { + return switch (schema.getType()) { + case NULL -> new NullSerializer(); + case STRING -> new StringSerializer(schema); + case ARRAY -> new ArraySerializer(schema); + case ENUM -> new EnumSerializer(schema); + case MAP -> new MapSerializer(schema); + case RECORD -> new RecordSerializer(schema); + default -> new GenericSerializer(); + }; + } +} From b8f6d59480389658ca79b7521b293def7350d821 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:43:31 +0530 Subject: [PATCH 05/47] Add visitor APIs to generate Avro messages for serialization --- .../lib/avro/visitor/ISerializeVisitor.java | 38 ++ .../lib/avro/visitor/SerializeVisitor.java | 348 ++++++++++++++++++ 2 files changed, 386 insertions(+) create mode 100644 native/src/main/java/io/ballerina/lib/avro/visitor/ISerializeVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/visitor/SerializeVisitor.java diff --git a/native/src/main/java/io/ballerina/lib/avro/visitor/ISerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/visitor/ISerializeVisitor.java new file mode 100644 index 0000000..557abc1 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/visitor/ISerializeVisitor.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.visitor; + +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +import java.util.Map; + +public interface ISerializeVisitor { + + String visitString(Object data); + GenericRecord visitRecord(BMap data, Schema schema) throws Exception; + Map visitMap(BMap data, Schema schema) throws Exception; + GenericData.Array visitArray(BArray data, Schema schema) throws Exception; + Object visitBytes(Object data, Schema schema); + Object visitEnum(Object data, Schema schema); + GenericData.Fixed visitFixed(Object data, Schema schema); +} diff --git a/native/src/main/java/io/ballerina/lib/avro/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/visitor/SerializeVisitor.java new file mode 100644 index 0000000..8bfa327 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/visitor/SerializeVisitor.java @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.visitor; + +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static io.ballerina.lib.avro.Utils.ARRAY_TYPE; +import static io.ballerina.lib.avro.Utils.FLOAT_TYPE; +import static io.ballerina.lib.avro.Utils.INTEGER_TYPE; +import static io.ballerina.lib.avro.Utils.MAP_TYPE; +import static io.ballerina.lib.avro.Utils.RECORD_TYPE; +import static io.ballerina.lib.avro.Utils.STRING_TYPE; + +public class SerializeVisitor implements ISerializeVisitor { + + @Override + public String visitString(Object data) { + return ((BString) data).getValue(); + } + + @Override + public GenericRecord visitRecord(BMap data, Schema schema) throws Exception { + GenericRecord genericRecord = new GenericData.Record(schema); + for (Schema.Field field: schema.getFields()) { + Object fieldData = data.get(StringUtils.fromString(field.name())); + Schema.Type type = field.schema().getType(); + switch (type) { + case UNION -> + genericRecord.put(field.name(), visitUnion(fieldData, field)); + case RECORD -> + genericRecord.put(field.name(), visitRecord((BMap) fieldData, field.schema())); + case MAP -> + genericRecord.put(field.name(), visitMap((BMap) fieldData, field.schema())); + case ARRAY -> + genericRecord.put(field.name(), visitArray((BArray) fieldData, field.schema())); + case INT, LONG, FLOAT, DOUBLE, BOOLEAN, STRING, BYTES -> + genericRecord.put(field.name(), visitRecordPrimitives(fieldData, type)); + case ENUM -> + genericRecord.put(field.name(), visitEnum(fieldData, field.schema())); + default -> + genericRecord.put(field.name(), fieldData); + } + } + return genericRecord; + } + + private Object visitRecordPrimitives(Object data, Schema.Type type) { + switch (type) { + case INT -> { + return ((Long) data).intValue(); + } + case BYTES -> { + ByteBuffer byteBuffer = ByteBuffer.allocate(((BArray) data).getByteArray().length); + byteBuffer.put(((BArray) data).getByteArray()); + byteBuffer.position(0); + return byteBuffer; + } + case STRING -> { + return data.toString(); + } + default -> { + return data; + } + } + } + + @Override + public Map visitMap(BMap data, Schema schema) throws Exception { + Map avroMap = new HashMap<>(); + if (schema.getType().equals(Schema.Type.UNION)) { + for (Schema fieldSchema: schema.getTypes()) { + if (fieldSchema.getType().equals(Schema.Type.MAP)) { + schema = fieldSchema; + } + } + } + Schema.Type type = schema.getValueType().getType(); + for (Object value : data.getKeys()) { + switch (type) { + case INT, LONG, FLOAT, DOUBLE, BOOLEAN, STRING, BYTES -> + avroMap.put(value.toString(), visitRecordPrimitives(data.get(value), type)); + case RECORD -> + avroMap.put(value.toString(), visitRecord((BMap) data.get(value), schema.getValueType())); + case MAP -> + avroMap.put(value.toString(), visitMap((BMap) data.get(value), schema.getValueType())); + case ARRAY -> + avroMap.put(value.toString(), visitArray((BArray) data.get(value), schema.getValueType())); + case ENUM -> + avroMap.put(value.toString(), visitEnum(data.get(value), schema.getValueType())); + case FIXED -> + avroMap.put(value.toString(), visitFixed(data.get(value), schema.getValueType())); + default -> + throw new IllegalArgumentException("Unsupported schema type: " + type); + } + } + return avroMap; + } + + @Override + public Object visitBytes(Object data, Schema schema) { + if (schema.getType().equals(Schema.Type.UNION)) { + for (Schema fieldSchema: schema.getTypes()) { + if (fieldSchema.getType().equals(Schema.Type.BYTES)) { + return ByteBuffer.wrap(((BArray) data).getByteArray()); + } + } + } + return ByteBuffer.wrap(((BArray) data).getByteArray()); + } + @Override + public Object visitEnum(Object data, Schema schema) { + return new GenericData.EnumSymbol(schema, data); + } + + @Override + public GenericData.Fixed visitFixed(Object data, Schema schema) { + return new GenericData.Fixed(schema, ((BArray) data).getByteArray()); + } + + @Override + public GenericData.Array visitArray(BArray data, Schema schema) throws Exception { + GenericData.Array array = new GenericData.Array<>(data.size(), schema); + switch (schema.getElementType().getType()) { + case ARRAY -> { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(value -> { + try { + array.add(visitArray((BArray) value, schema.getElementType())); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } + case ENUM -> { + Arrays.stream((data.getValues() == null) ? data.getStringArray() : data.getValues()) + .filter(Objects::nonNull) + .forEach(value -> { + try { + array.add(new GenericData.EnumSymbol(schema.getElementType(), value)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } + case STRING -> { + return visitStringArray(data, array); + } + case INT, LONG -> { + return visitLongArray(data, array); + } + case FLOAT, DOUBLE -> { + return visitDoubleArray(data, array); + } + case BOOLEAN -> { + return visitBooleanArray(data, array); + } + case MAP -> { + return visitMapArray(data, schema, array); + } + case RECORD -> { + return visitRecordArray(data, schema, array); + } + case FIXED -> { + return visitFixed(data, array, schema.getElementType()); + } + default -> { + return visitBytes(data, array); + } + } + } + + private static GenericData.Array visitRecordArray(BArray data, Schema schema, + GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(record -> { + try { + array.add(new SerializeVisitor().visitRecord((BMap) record, schema.getElementType())); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } + + private static GenericData.Array visitMapArray(BArray data, Schema schema, + GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(record -> { + try { + array.add(new SerializeVisitor().visitMap((BMap) record, schema.getElementType())); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } + + private GenericData.Array visitBytes(BArray data, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(bytes -> { + ByteBuffer byteBuffer = ByteBuffer.allocate(((BArray) bytes).getByteArray().length); + byteBuffer.put(((BArray) bytes).getByteArray()); + byteBuffer.position(0); + array.add(byteBuffer); + }); + return array; + } + + private GenericData.Array visitFixed(BArray data, GenericData.Array array, Schema schema) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(bytes -> { + GenericFixed genericFixed = new GenericData.Fixed(schema, ((BArray) bytes).getByteArray()); + array.add(genericFixed); + }); + return array; + } + + private GenericData.Array visitBooleanArray(BArray data, GenericData.Array array) { + for (Object obj: data.getBooleanArray()) { + array.add(obj); + } + return array; + } + + private GenericData.Array visitDoubleArray(BArray data, GenericData.Array array) throws Exception { + try { + for (Object obj: data.getFloatArray()) { + array.add(obj); + } + return array; + } catch (Exception e) { + throw new Exception(e.getMessage()); + } + } + + private GenericData.Array visitLongArray(BArray data, GenericData.Array array) { + for (Object obj: data.getIntArray()) { + array.add(obj); + } + return array; + } + + private GenericData.Array visitStringArray(BArray data, GenericData.Array array) { + array.addAll(Arrays.asList(data.getStringArray())); + return array; + } + + private Object visitUnion(Object data, Schema.Field field) throws Exception { + Type typeName = TypeUtils.getType(data); + switch (typeName.getClass().getSimpleName()) { + case STRING_TYPE -> { + for (Schema schema : field.schema().getTypes()) { + if (schema.getType().equals(Schema.Type.ENUM)) { + return new GenericData.EnumSymbol(schema, data); + } + } + return data.toString(); + } + case ARRAY_TYPE -> { + for (Schema schema: field.schema().getTypes()) { + if (schema.getType().equals(Schema.Type.BYTES)) { + return ByteBuffer.wrap(((BArray) data).getByteArray()); + } else if (schema.getType().equals(Schema.Type.FIXED)) { + return new GenericData.Fixed(schema, ((BArray) data).getByteArray()); + } else if (schema.getType().equals(Schema.Type.ARRAY)) { + return visitArray((BArray) data, schema); + } + } + return visitArray((BArray) data, field.schema()); + } + case MAP_TYPE -> { + return visitMap((BMap) data, field.schema()); + } + case RECORD_TYPE -> { + return visitRecord((BMap) data, getRecordSchema(Schema.Type.RECORD, field.schema().getTypes())); + } + case INTEGER_TYPE -> { + for (Schema schema : field.schema().getTypes()) { + if (schema.getType().equals(Schema.Type.INT)) { + return ((Long) data).intValue(); + } + } + return data; + } + case FLOAT_TYPE -> { + for (Schema schema: field.schema().getTypes()) { + if (schema.getType().equals(Schema.Type.FLOAT)) { + return ((Double) data).floatValue(); + } + } + return data; + } + default -> { + return data; + } + } + } + + public static Schema getRecordSchema(Schema.Type givenType, List schemas) { + for (Schema schema: schemas) { + if (schema.getType().equals(Schema.Type.UNION)) { + getRecordSchema(givenType, schema.getTypes()); + } else if (schema.getType().equals(givenType)) { + return schema; + } + } + return null; + } +} From d010c1338178e8c2bffc10455cceb63f9218043a Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:44:19 +0530 Subject: [PATCH 06/47] Add visitable classes for each Avro schema type in deserialization --- .../avro/deserialize/ArrayDeserializer.java | 36 ++++++++++++ .../lib/avro/deserialize/Deserializer.java | 55 +++++++++++++++++++ .../avro/deserialize/DoubleDeserializer.java | 30 ++++++++++ .../avro/deserialize/GenericDeserializer.java | 30 ++++++++++ .../lib/avro/deserialize/MapDeserializer.java | 37 +++++++++++++ .../avro/deserialize/NullDeserializer.java | 33 +++++++++++ .../avro/deserialize/RecordDeserializer.java | 36 ++++++++++++ .../avro/deserialize/StringDeserializer.java | 30 ++++++++++ 8 files changed, 287 insertions(+) create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java new file mode 100644 index 0000000..a84d49e --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public class ArrayDeserializer extends Deserializer { + + public ArrayDeserializer(Schema schema, Type type) { + super(schema, type); + } + + @Override + public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception { + return visitor.visitArray(getSchema(), (GenericData.Array) data, getType()); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java new file mode 100644 index 0000000..eede4b9 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.TypeUtils; +import org.apache.avro.Schema; + +public abstract class Deserializer { + + private final Schema schema; + private final Type type; + + public Deserializer() { + this.schema = null; + this.type = null; + } + + public Deserializer(Type type) { + this.schema = null; + this.type = TypeUtils.getReferredType(type); + } + + public Deserializer(Schema schema, Type type) { + this.schema = new Schema.Parser().parse(schema.toString()); + this.type = TypeUtils.getReferredType(type); + } + + protected Schema getSchema() { + return schema; + } + + protected Type getType() { + return type; + } + + public abstract Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception; +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java new file mode 100644 index 0000000..9f1482e --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import org.apache.avro.Schema; + +public class DoubleDeserializer extends Deserializer { + + @Override + public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) { + return visitor.visitDouble(data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java new file mode 100644 index 0000000..d2242b0 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import org.apache.avro.Schema; + +public class GenericDeserializer extends Deserializer { + + @Override + public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) { + return data; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java new file mode 100644 index 0000000..f04df55 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; + +import java.util.Map; + +public class MapDeserializer extends Deserializer { + + public MapDeserializer(Type type) { + super(type); + } + + @Override + public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception { + return visitor.visitMap((Map) data, getType(), schema); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java new file mode 100644 index 0000000..91f0801 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import org.apache.avro.Schema; + +public class NullDeserializer extends Deserializer { + + @Override + public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception { + if (data != null) { + throw new Exception("The value does not match with the null schema"); + } + return null; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java new file mode 100644 index 0000000..17bf236 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; + +public class RecordDeserializer extends Deserializer { + + public RecordDeserializer(Schema schema, Type type) { + super(schema, type); + } + + @Override + public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception { + return visitor.visitRecords(getType(), getSchema(), (GenericRecord) data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java new file mode 100644 index 0000000..b661c85 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import org.apache.avro.Schema; + +public class StringDeserializer extends Deserializer { + + @Override + public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) { + return visitor.visitString(data); + } +} From 9dd6f82251eea214044e006008e33511e491ba46 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:44:58 +0530 Subject: [PATCH 07/47] Add factory pattern to generate Avro messages from deserialized data --- .../main/java/io/ballerina/lib/avro/Avro.java | 30 ++++---------- .../avro/deserialize/DeserializeFactory.java | 39 +++++++++++++++++++ 2 files changed, 47 insertions(+), 22 deletions(-) create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java diff --git a/native/src/main/java/io/ballerina/lib/avro/Avro.java b/native/src/main/java/io/ballerina/lib/avro/Avro.java index a8d8bd1..ea65f32 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Avro.java +++ b/native/src/main/java/io/ballerina/lib/avro/Avro.java @@ -76,29 +76,15 @@ public static Object toAvro(BObject schemaObject, Object data) { public static Object fromAvro(BObject schemaObject, BArray payload, BTypedesc typeParam) { Schema schema = (Schema) schemaObject.getNativeData(AVRO_SCHEMA); - byte[] avroBytes = payload.getByteArray(); - if (Schema.Type.FIXED.equals(schema.getType())) { - return ValueUtils.convert(ValueCreator.createArrayValue(avroBytes), typeParam.getDescribingType()); - } - JsonNode deserializedJsonString; + DatumReader datumReader = new GenericDatumReader<>(schema); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(payload.getBytes(), null); try { - AvroMapper mapper = new AvroMapper(); - deserializedJsonString = mapper.readerFor(Object.class).with(new AvroSchema(schema)).readTree(avroBytes); - } catch (IOException e) { - return Utils.createError(DESERIALIZATION_ERROR, e); - } - Object jsonObject = JsonUtils.parse(deserializedJsonString.toPrettyString()); - return ValueUtils.convert(jsonObject, typeParam.getDescribingType()); - } - - private static Object generateJsonObject(Object data, Schema schema, - ObjectMapper objectMapper) throws JsonProcessingException { - if (Schema.Type.NULL.equals(schema.getType()) || Schema.Type.FIXED.equals(schema.getType())) { - return data; - } else if (Schema.Type.STRING.equals(schema.getType()) || Schema.Type.ENUM.equals(schema.getType())) { - return objectMapper.readValue("\"" + data + "\"", Object.class); + Object data = datumReader.read(payload, decoder); + DeserializeVisitor deserializeVisitor = new DeserializeVisitor(); + Deserializer deserializer = DeserializeFactory.generateDeserializer(schema, typeParam.getDescribingType()); + return Objects.requireNonNull(deserializer).fromAvroMessage(deserializeVisitor, data, schema); + } catch (Exception e) { + return createError(DESERIALIZATION_ERROR, e); } - Object jsonString = JsonUtils.parse(StringUtils.getJsonString(data)); - return objectMapper.readValue(jsonString.toString(), Object.class); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java new file mode 100644 index 0000000..a98f81d --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; + +public class DeserializeFactory { + + public static Deserializer generateDeserializer(Schema schema, Type type) { + return switch (schema.getType()) { + case NULL -> new NullDeserializer(); + case FLOAT, DOUBLE -> new DoubleDeserializer(); + case STRING, ENUM -> new StringDeserializer(); + case ARRAY -> new ArrayDeserializer(schema, type); + case FIXED -> new FixedDeserializer(); + case MAP -> new MapDeserializer(type); + case RECORD -> new RecordDeserializer(schema, type); + case BYTES -> new ByteDeserializer(); + default -> new GenericDeserializer(); + }; + } +} From 05ecab3ad7b224171bac252ef9ca0bf6732944ec Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:46:27 +0530 Subject: [PATCH 08/47] Add visitor APIs to generate deserialized message --- .../lib/avro/visitor/DeserializeVisitor.java | 411 ++++++++++++++++++ .../lib/avro/visitor/IDeserializeVisitor.java | 39 ++ 2 files changed, 450 insertions(+) create mode 100644 native/src/main/java/io/ballerina/lib/avro/visitor/DeserializeVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/visitor/IDeserializeVisitor.java diff --git a/native/src/main/java/io/ballerina/lib/avro/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/visitor/DeserializeVisitor.java new file mode 100644 index 0000000..a6c63ce --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/visitor/DeserializeVisitor.java @@ -0,0 +1,411 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.visitor; + +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.ArrayType; +import io.ballerina.runtime.api.types.Field; +import io.ballerina.runtime.api.types.IntersectionType; +import io.ballerina.runtime.api.types.MapType; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericEnumSymbol; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.util.Utf8; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static io.ballerina.lib.avro.Utils.getMutableType; + +public class DeserializeVisitor implements IDeserializeVisitor { + + public double visitDouble(Object data) { + if (data instanceof Float) { + return Double.parseDouble(data.toString()); + } + return (double) data; + } + + public BArray visitBytes(Object data) { + return ValueCreator.createArrayValue(((ByteBuffer) data).array()); + } + + public BArray visitFixed(Object data) { + GenericData.Fixed fixed = (GenericData.Fixed) data; + return ValueCreator.createArrayValue(fixed.bytes()); + } + + public BString visitString(Object data) { + return StringUtils.fromString(data.toString()); + } + + @SuppressWarnings({"unchecked", "deprecation"}) + public BMap visitMap(Map data, Type type, Schema schema) throws Exception { + assert type instanceof MapType; + BMap avroRecord = ValueCreator.createMapValue(type); + Object[] keys = data.keySet().toArray(); + for (Object key : keys) { + Object value = data.get(key); + Schema.Type valueType = schema.getValueType().getType(); + switch (valueType) { + case ARRAY -> + avroRecord.put(StringUtils.fromString(key.toString()), visitArray(schema.getValueType(), + (GenericData.Array) value, ((MapType) type).getConstrainedType())); + case BYTES -> + avroRecord.put(StringUtils.fromString(key.toString()), + ValueCreator.createArrayValue(((ByteBuffer) value).array())); + case FIXED -> + avroRecord.put(StringUtils.fromString(key.toString()), + ValueCreator.createArrayValue(((GenericFixed) value).bytes())); + case RECORD -> + avroRecord.put(StringUtils.fromString(key.toString()), + visitRecords(((MapType) type).getConstrainedType().getCachedReferredType(), + schema.getValueType(), (GenericRecord) value)); + case ENUM, STRING -> + avroRecord.put(StringUtils.fromString(key.toString()), + StringUtils.fromString(value.toString())); + case FLOAT -> + avroRecord.put(StringUtils.fromString(key.toString()), Double.parseDouble(value.toString())); + case MAP -> + avroRecord.put(StringUtils.fromString(key.toString()), + visitMap((Map) value, + ((MapType) type).getConstrainedType(), schema.getValueType())); + default -> + avroRecord.put(StringUtils.fromString(key.toString()), value); + } + } + return avroRecord; + } + + public Object visitArray(Schema schema, GenericData.Array data, Type type) throws Exception { + switch (schema.getElementType().getType()) { + case ARRAY -> { + Object[] objects = new Object[data.size()]; + Type arrayType = ((ArrayType) type).getElementType(); + for (int i = 0; i < data.size(); i++) { + objects[i] = visitArray(schema.getElementType(), + (GenericData.Array) data.get(i), arrayType); + } + return ValueCreator.createArrayValue(objects, (ArrayType) type); + } + case STRING -> { + return visitStringArray(data); + } + case ENUM -> { + Object[] enums = new Object[data.size()]; + for (int i = 0; i < data.size(); i++) { + enums[i] = visitString(data.get(i)); + } + return ValueCreator.createArrayValue(enums, (ArrayType) type); + } + case INT -> { + return visitIntArray(data); + } + case LONG -> { + return visitLongArray(data); + } + case FLOAT, DOUBLE -> { + return visitDoubleArray(data); + } + case BOOLEAN -> { + return visitBooleanArray(data); + } + case RECORD -> { + return visitRecordArray(schema, data, type); + } + case FIXED -> { + assert type instanceof ArrayType; + return visitFixedArray(data, (ArrayType) type); + } + default -> { + assert type instanceof ArrayType; + return visitBytesArray(data, (ArrayType) type); + } + } + } + + private BArray visitBytesArray(GenericData.Array data, ArrayType type) { + List values = new ArrayList<>(); + for (Object datum : data) { + values.add(visitBytes(datum)); + } + return ValueCreator.createArrayValue(values.toArray(new BArray[data.size()]), type); + } + + private BArray visitFixedArray(GenericData.Array data, ArrayType type) { + List values = new ArrayList<>(); + for (Object datum : data) { + values.add(visitFixed(datum)); + } + return ValueCreator.createArrayValue(values.toArray(new BArray[data.size()]), type); + } + + private BArray visitRecordArray(Schema schema, GenericData.Array data, Type type) throws Exception { + List recordList = new ArrayList<>(); + if (type instanceof ArrayType arrayType) { + for (Object datum : data) { + recordList.add(visitRecords(arrayType.getElementType().getCachedReferredType(), + schema.getElementType(), (GenericRecord) datum)); + } + } + assert type instanceof ArrayType; + return ValueCreator.createArrayValue(recordList.toArray(new Object[data.size()]), (ArrayType) type); + + } + + private static BArray visitBooleanArray(GenericData.Array data) { + boolean[] booleanArray = new boolean[data.size()]; + int index = 0; + for (Object datum : data) { + booleanArray[index++] = (boolean) datum; + } + return ValueCreator.createArrayValue(booleanArray); + } + + private BArray visitDoubleArray(GenericData.Array data) { + List doubleList = new ArrayList<>(); + for (Object datum : data) { + doubleList.add(visitDouble(datum)); + } + double[] doubleArray = doubleList.stream().mapToDouble(Double::doubleValue).toArray(); + return ValueCreator.createArrayValue(doubleArray); + } + + private static BArray visitLongArray(GenericData.Array data) { + List longList = new ArrayList<>(); + for (Object datum : data) { + longList.add((Long) datum); + } + long[] longArray = longList.stream().mapToLong(Long::longValue).toArray(); + return ValueCreator.createArrayValue(longArray); + } + + private static BArray visitIntArray(GenericData.Array data) { + List longList = new ArrayList<>(); + for (Object datum : data) { + longList.add(((Integer) datum).longValue()); + } + long[] longArray = longList.stream().mapToLong(Long::longValue).toArray(); + return ValueCreator.createArrayValue(longArray); + } + + private BArray visitStringArray(GenericData.Array data) { + BString[] stringArray = new BString[data.size()]; + for (int i = 0; i < data.size(); i++) { + stringArray[i] = visitString(data.get(i)); + } + return ValueCreator.createArrayValue(stringArray); + } + + @SuppressWarnings("unchecked") + public BMap visitRecords(Type type, Schema schema, GenericRecord rec) throws Exception { + BMap avroRecord; + Type originalType = type; + if (type instanceof IntersectionType intersectionType) { + type = getMutableType(intersectionType); + avroRecord = ValueCreator.createRecordValue((RecordType) type); + } else if (type instanceof RecordType recordType) { + avroRecord = ValueCreator.createRecordValue(recordType); + } else { + throw new Exception("Type is not a valid record type"); + } + for (Schema.Field field : schema.getFields()) { + Object fieldData = rec.get(field.name()); + switch (field.schema().getType()) { + case MAP -> { + Type mapType = extractMapType(type); + avroRecord.put(StringUtils.fromString(field.name()), + visitMap((Map) rec.get(field.name()), mapType, field.schema())); + } + case ARRAY -> + avroRecord.put(StringUtils.fromString(field.name()), visitArray(field.schema(), + (GenericData.Array) rec.get(field.name()), type)); + case BYTES -> { + ByteBuffer byteBuffer = (ByteBuffer) rec.get(field.name()); + avroRecord.put(StringUtils.fromString(field.name()), + ValueCreator.createArrayValue(byteBuffer.array())); + } + case STRING -> + avroRecord.put(StringUtils.fromString(field.name()), + StringUtils.fromString(rec.get(field.name()).toString())); + case RECORD -> { + Type recType = extractRecordType((RecordType) type); + avroRecord.put(StringUtils.fromString(field.name()), + visitRecords(recType, field.schema(), (GenericRecord) rec.get(field.name()))); + } + case INT -> + avroRecord.put(StringUtils.fromString(field.name()), Long.parseLong(fieldData.toString())); + case FLOAT -> + avroRecord.put(StringUtils.fromString(field.name()), Double.parseDouble(fieldData.toString())); + case UNION -> + visitUnionRecords(type, avroRecord, field, fieldData); + default -> + avroRecord.put(StringUtils.fromString(field.name()), rec.get(field.name())); + } + } + if (originalType.isReadOnly()) { + avroRecord.freezeDirect(); + } + return avroRecord; + } + + @SuppressWarnings("unchecked") + private void visitUnionRecords(Type type, BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + for (Schema schemaType : field.schema().getTypes()) { + if (fieldData == null) { + avroRecord.put(StringUtils.fromString(field.name()), null); + break; + } + switch (schemaType.getType()) { + case BYTES -> { + if (fieldData instanceof ByteBuffer) { + BArray byteArray = ValueCreator.createArrayValue(((ByteBuffer) fieldData).array()); + avroRecord.put(StringUtils.fromString(field.name()), byteArray); + } + } + case FIXED -> { + if (fieldData instanceof GenericFixed) { + BArray byteArray = ValueCreator.createArrayValue(((GenericData.Fixed) fieldData).bytes()); + avroRecord.put(StringUtils.fromString(field.name()), byteArray); + } + } + case ARRAY -> { + if (fieldData instanceof GenericData.Array) { + Object[] objectArray = ((GenericData.Array) fieldData).toArray(); + if (schemaType.getElementType().getType().equals(Schema.Type.STRING) + || schemaType.getElementType().getType().equals(Schema.Type.ENUM)) { + BString[] stringArray = new BString[objectArray.length]; + BArray ballerinaArray = ValueCreator.createArrayValue(stringArray); + int i = 0; + for (Object obj : objectArray) { + stringArray[i] = StringUtils.fromString(obj.toString()); + i++; + } + avroRecord.put(StringUtils.fromString(field.name()), ballerinaArray); + } else { + avroRecord.put(StringUtils.fromString(field.name()), fieldData); + } + } + } + case MAP -> { + if (fieldData instanceof Map) { + BMap avroMap = ValueCreator.createMapValue(); + Object[] keys = ((Map) fieldData).keySet().toArray(); + for (Object key : keys) { + avroMap.put(StringUtils.fromString(key.toString()), + ((Map) fieldData).get(key)); + } + avroRecord.put(StringUtils.fromString(field.name()), avroMap); + } + } + case RECORD -> { + if (fieldData instanceof GenericRecord) { + avroRecord.put(StringUtils.fromString(field.name()), + visitRecords(type, schemaType, (GenericRecord) fieldData)); + } + } + case STRING -> { + if (fieldData instanceof Utf8) { + avroRecord.put(StringUtils.fromString(field.name()), + StringUtils.fromString(fieldData.toString())); + } + } + case INT, LONG -> { + if (fieldData instanceof Integer || fieldData instanceof Long) { + avroRecord.put(StringUtils.fromString(field.name()), + ((Number) fieldData).longValue()); + } + } + case FLOAT, DOUBLE -> { + if (fieldData instanceof Double) { + avroRecord.put(StringUtils.fromString(field.name()), fieldData); + } else { + avroRecord.put(StringUtils.fromString(field.name()), Double.parseDouble(fieldData.toString())); + } + } + case ENUM -> { + if (fieldData instanceof GenericEnumSymbol) { + avroRecord.put(StringUtils.fromString(field.name()), + StringUtils.fromString(fieldData.toString())); + } + } + default -> { + if (fieldData instanceof Boolean) { + avroRecord.put(StringUtils.fromString(field.name()), fieldData); + } + } + } + } + } + + private static Type extractMapType(Type type) { + Type mapType = type; + for (Map.Entry entry : ((RecordType) type).getFields().entrySet()) { + Field fieldValue = entry.getValue(); + if (fieldValue != null) { + Type fieldType = fieldValue.getFieldType(); + if (fieldType instanceof MapType) { + mapType = fieldType; + } else if (TypeUtils.getReferredType(fieldType) instanceof MapType) { + mapType = TypeUtils.getReferredType(fieldType); + } else if (fieldType instanceof IntersectionType) { + Type referredType = getMutableType((IntersectionType) fieldType); + if (referredType instanceof MapType) { + mapType = referredType; + } + } + } + } + return mapType; + } + + private static RecordType extractRecordType(RecordType type) { + Map fieldsMap = type.getFields(); + RecordType recType = type; + for (Map.Entry entry : fieldsMap.entrySet()) { + Field fieldValue = entry.getValue(); + if (fieldValue != null) { + Type fieldType = fieldValue.getFieldType(); + if (fieldType instanceof RecordType) { + recType = (RecordType) fieldType; + } else if (fieldType instanceof IntersectionType) { + Type getType = getMutableType((IntersectionType) fieldType); + if (getType instanceof RecordType) { + recType = (RecordType) getType; + } + } else if (TypeUtils.getReferredType(fieldType) instanceof RecordType) { + recType = (RecordType) TypeUtils.getReferredType(fieldType); + } + } + } + return recType; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/visitor/IDeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/visitor/IDeserializeVisitor.java new file mode 100644 index 0000000..153466a --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/visitor/IDeserializeVisitor.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.visitor; + +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +import java.util.Map; + +public interface IDeserializeVisitor { + + public double visitDouble(Object data); + public BString visitString(Object data); + BMap visitMap(Map data, Type type, Schema schema) throws Exception; + BArray visitFixed(Object data); + Object visitArray(Schema schema, GenericData.Array data, Type type) throws Exception; + BMap visitRecords(Type type, Schema schema, GenericRecord rec) throws Exception; +} From 7e9c235454069781e8d652026fd15c91ec3055de Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:47:17 +0530 Subject: [PATCH 09/47] Add visitable for the string type --- .../lib/avro/serialize/StringSerializer.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java new file mode 100644 index 0000000..035e42c --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.visitor.SerializeVisitor; +import org.apache.avro.Schema; + +public class StringSerializer extends Serializer { + + public StringSerializer(Schema schema) { + super(schema); + } + + @Override + public Object generateMessage(SerializeVisitor serializeVisitor, Object data) { + return serializeVisitor.visitString(data); + } +} From f423aaf4d35a5252f79e0e06f4851f8bc0584d4b Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:47:37 +0530 Subject: [PATCH 10/47] Add support for fixed and byte values --- .../avro/deserialize/ByteDeserializer.java | 11 ++++++ .../avro/deserialize/FixedDeserializer.java | 30 ++++++++++++++++ .../lib/avro/serialize/ByteSerializer.java | 13 +++++++ .../lib/avro/serialize/FixedSerializer.java | 34 +++++++++++++++++++ .../lib/avro/serialize/MessageFactory.java | 2 ++ 5 files changed, 90 insertions(+) create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java new file mode 100644 index 0000000..7825dfe --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java @@ -0,0 +1,11 @@ +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import org.apache.avro.Schema; + +public class ByteDeserializer extends Deserializer { + @Override + public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception { + return visitor.visitBytes(data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java new file mode 100644 index 0000000..95eba4c --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import org.apache.avro.Schema; + +public class FixedDeserializer extends Deserializer { + + @Override + public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) { + return visitor.visitFixed(data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java new file mode 100644 index 0000000..7ea67f5 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java @@ -0,0 +1,13 @@ +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BArray; + +import java.nio.ByteBuffer; + +public class ByteSerializer extends Serializer { + @Override + public Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception { + return ByteBuffer.wrap(((BArray) data).getByteArray()); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java new file mode 100644 index 0000000..ad4af55 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.visitor.SerializeVisitor; +import org.apache.avro.Schema; + +public class FixedSerializer extends Serializer { + + public FixedSerializer(Schema schema) { + super(schema); + } + + @Override + public Object generateMessage(SerializeVisitor serializeVisitor, Object data) { + return serializeVisitor.visitFixed(data, getSchema()); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java index 150b1b4..c9daf44 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java @@ -27,9 +27,11 @@ public static Serializer createMessage(Schema schema) { case NULL -> new NullSerializer(); case STRING -> new StringSerializer(schema); case ARRAY -> new ArraySerializer(schema); + case FIXED -> new FixedSerializer(schema); case ENUM -> new EnumSerializer(schema); case MAP -> new MapSerializer(schema); case RECORD -> new RecordSerializer(schema); + case BYTES -> new ByteSerializer(); default -> new GenericSerializer(); }; } From c96bd7739a675aaf740df3770003986b29053381 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:48:07 +0530 Subject: [PATCH 11/47] Add API to extract mutable type from an intersection type --- .../java/io/ballerina/lib/avro/Utils.java | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/Utils.java b/native/src/main/java/io/ballerina/lib/avro/Utils.java index 7b7fc5b..00cfa3a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Utils.java +++ b/native/src/main/java/io/ballerina/lib/avro/Utils.java @@ -19,7 +19,12 @@ package io.ballerina.lib.avro; import io.ballerina.runtime.api.creators.ErrorCreator; +import io.ballerina.runtime.api.types.IntersectionType; +import io.ballerina.runtime.api.types.MapType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.types.UnionType; import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; import io.ballerina.runtime.api.values.BError; import static io.ballerina.lib.avro.ModuleUtils.getModule; @@ -31,11 +36,34 @@ private Utils() { public static final String AVRO_SCHEMA = "avroSchema"; public static final String ERROR_TYPE = "Error"; - public static final String JSON_PROCESSING_ERROR = "JSON processing error"; - public static final String DESERIALIZATION_ERROR = "Unable to deserialize the byte value"; + public static final String SERIALIZATION_ERROR = "Avro serialization error"; + public static final String DESERIALIZATION_ERROR = "Avro deserialization error"; + public static final String STRING_TYPE = "BStringType"; + public static final String ARRAY_TYPE = "BArrayType"; + public static final String MAP_TYPE = "BMapType"; + public static final String RECORD_TYPE = "BRecordType"; + public static final String INTEGER_TYPE = "BIntegerType"; + public static final String FLOAT_TYPE = "BFloatType"; public static BError createError(String message, Throwable throwable) { BError cause = ErrorCreator.createError(throwable); return ErrorCreator.createError(getModule(), ERROR_TYPE, StringUtils.fromString(message), cause, null); } + + public static Type getMutableType(IntersectionType intersectionType) { + for (Type type : intersectionType.getConstituentTypes()) { + Type referredType = TypeUtils.getImpliedType(type); + if (referredType instanceof UnionType) { + for (Type elementType : ((UnionType) referredType).getMemberTypes()) { + if (elementType instanceof MapType) { + return elementType; + } + } + } + if (TypeUtils.getImpliedType(intersectionType.getEffectiveType()).getTag() == referredType.getTag()) { + return referredType; + } + } + throw new IllegalStateException("Unsupported intersection type found: " + intersectionType); + } } From 2941907875c1d1018ac33a52d1b62570df43a69e Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:48:37 +0530 Subject: [PATCH 12/47] Remove load test workflow file --- .github/workflows/process-load-test-result.yml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 .github/workflows/process-load-test-result.yml diff --git a/.github/workflows/process-load-test-result.yml b/.github/workflows/process-load-test-result.yml deleted file mode 100644 index 10fd790..0000000 --- a/.github/workflows/process-load-test-result.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Process load test results -on: - repository_dispatch: - types: [ avro-load-test ] - -jobs: - call_stdlib_process_load_test_results_workflow: - name: Run StdLib Process Load Test Results Workflow - uses: ballerina-platform/ballerina-library/.github/workflows/process-load-test-results-template.yml@main - with: - results: ${{ toJson(github.event.client_payload.results) }} - secrets: - ballerina_bot_token: ${{ secrets.BALLERINA_BOT_TOKEN }} - ballerina_reviewer_bot_token: ${{ secrets.BALLERINA_REVIEWER_BOT_TOKEN }} From f10faa4c74d9d8fad2963a70c158c055c91ae879 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:49:05 +0530 Subject: [PATCH 13/47] Add test cases to validate the functioanlity --- ballerina/tests/primitive_tests.bal | 161 ++++++++ ballerina/tests/record_tests.bal | 614 ++++++++++++++++++++++++++++ ballerina/tests/test.bal | 392 ++---------------- ballerina/tests/types.bal | 76 +++- 4 files changed, 879 insertions(+), 364 deletions(-) create mode 100644 ballerina/tests/primitive_tests.bal create mode 100644 ballerina/tests/record_tests.bal diff --git a/ballerina/tests/primitive_tests.bal b/ballerina/tests/primitive_tests.bal new file mode 100644 index 0000000..3f7d43b --- /dev/null +++ b/ballerina/tests/primitive_tests.bal @@ -0,0 +1,161 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; + +@test:Config { + groups: ["primitive", "int"] +} +public isolated function testIntValue() returns error? { + string schema = string ` + { + "type": "int", + "name" : "intValue", + "namespace": "data" + }`; + + int value = 5; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(value); + int deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, value); +} + +@test:Config { + groups: ["primitive", "float"] +} +public isolated function testFloatValue() returns error? { + string schema = string ` + { + "type": "float", + "name" : "floatValue", + "namespace": "data" + }`; + + float value = 5.5; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(value); + float deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, value); +} + +@test:Config { + groups: ["primitive", "double"] +} +public isolated function testDoubleValue() returns error? { + string schema = string ` + { + "type": "double", + "name" : "doubleValue", + "namespace": "data" + }`; + + float value = 5.5595; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(value); + float deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, value); +} + +@test:Config { + groups: ["primitive", "check", "l"] +} +public isolated function testLongValue() returns error? { + string schema = string ` + { + "type": "long", + "name" : "longValue", + "namespace": "data" + }`; + + int value = 555950000000000000; + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(value); + int deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, value); +} + +@test:Config { + groups: ["primitive", "check"] +} +public isolated function testStringValue() returns error? { + string schema = string ` + { + "type": "string", + "name" : "stringValue", + "namespace": "data" + }`; + + string value = "test"; + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(value); + string deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, value); +} + +@test:Config { + groups: ["primitive", "check"] +} +public isolated function testBoolean() returns error? { + string schema = string ` + { + "type": "boolean", + "name" : "booleanValue", + "namespace": "data" + }`; + + boolean value = true; + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(value); + boolean deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, value); +} + +@test:Config { + groups: ["primitive", "null"] +} +public isolated function testNullValues() returns error? { + string schema = string ` + { + "type": "null", + "name" : "nullValue", + "namespace": "data" + }`; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(()); + () deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, ()); +} + +@test:Config { + groups: ["primitive", "null"] +} +public isolated function testNullValuesWithNonNullData() returns error? { + string schema = string ` + { + "type": "null", + "name" : "nullValue", + "namespace": "data" + }`; + + Schema avro = check new (schema); + byte[]|error encode = avro.toAvro("string"); + test:assertTrue(encode is error); +} diff --git a/ballerina/tests/record_tests.bal b/ballerina/tests/record_tests.bal new file mode 100644 index 0000000..17903c4 --- /dev/null +++ b/ballerina/tests/record_tests.bal @@ -0,0 +1,614 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; + +@test:Config { + groups: ["record"] +} +public isolated function testRecords() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "subject", "type": "string"} + ] + }`; + + Student student = { + name: "Liam", + subject: "geology" + }; + + Schema avro = check new (schema); + byte[] serialize = check avro.toAvro(student); + Student deserialize = check avro.fromAvro(serialize); + test:assertEquals(student, deserialize); +} + +@test:Config { + groups: ["record"] +} +public isolated function testRecordsWithDifferentTypeOfFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "int"} + ] + }`; + + Person student = { + name: "Liam", + age: 52 + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(student); + Person deserialize = check avro.fromAvro(encode); + test:assertEquals(student, deserialize); +} + +@test:Config { + groups: ["record"] +} +public isolated function testNestedRecords() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": { + "type": "map", + "values" : "int", + "default": {} + } + }, + { + "name": "age", + "type": "long" + }, + { + "name": "instructor", + "type": { + "name": "Instructor", + "type": "record", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "student", + "type": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + } + } + ] + } + } + ] + }`; + + Lecturer3 lecturer = { + name: {"John": 1, "Sam": 2, "Liam": 3}, + age: 11, + instructor: { + name: "Liam", + student: { + name: "Sam", + subject: "geology" + } + } + }; + + Schema avro = check new (schema); + byte[] serialize = check avro.toAvro(lecturer); + Lecturer3 deserialize = check avro.fromAvro(serialize); + // deserialize.instructor.student.name = "Sam"; + test:assertEquals(deserialize, lecturer); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testArraysInRecords() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "colors", "type": {"type": "array", "items": "string"}} + ] + }`; + + Color colors = { + name: "Red", + colors: ["maroon", "dark red", "light red"] + }; + + Schema avro = check new (schema); + byte[] serialize = check avro.toAvro(colors); + Color deserialize = check avro.fromAvro(serialize); + test:assertEquals(colors, deserialize); +} + +type Color1 record { + string name; + byte[] colors; +}; + +@test:Config { + groups: ["record", "errors"] +} +public isolated function testArraysInRecordsWithInvalidSchema() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "colors", "type": "bytes"} + ] + }`; + + Color1 colors = { + name: "Red", + colors: "ss".toBytes() + }; + + Schema avroProducer = check new (schema); + byte[] serialize = check avroProducer.toAvro(colors); + string schema2 = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "colors", "type": {"type": "array", "items": "int"}} + ] + }`; + Schema avroConsumer = check new (schema2); + Color1|Error deserialize = avroConsumer.fromAvro(serialize); + test:assertTrue(deserialize is Error); +} + +@test:Config { + groups: ["record", "union"] +} +public isolated function testRecordsWithUnionTypes() returns error? { + string schema = string ` + { + "type": "record", + "name": "Course", + "namespace": "example.avro", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "value", + "type": "float" + }, + { + "name": "credits", + "type": ["null", "int"] + }, + { + "name": "student", + "type": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + } + ] + }`; + + UnionRecord course = { + name: "data", + value: 0.0, + credits: 5, + student: {name: "Jon", subject: "geo"} + }; + + Schema avro = check new (schema); + byte[] serialize = check avro.toAvro(course); + UnionRecord deserialize = check avro.fromAvro(serialize); + test:assertEquals(deserialize, course); +} + +@test:Config { + groups: ["record", "primitive", "int"] +} +public isolated function testRecordsWithIntFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "int"} + ] + }`; + + Person student = { + name: "Liam", + age: 52 + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(student); + Person deserialize = check avro.fromAvro(encode); + test:assertEquals(student, deserialize); +} + +@test:Config { + groups: ["record", "primitive", "long"] +} +public isolated function testRecordsWithLongFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "long"} + ] + }`; + + Person student = { + name: "Liam", + age: 52 + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(student); + Person deserialize = check avro.fromAvro(encode); + test:assertEquals(student, deserialize); +} + +@test:Config { + groups: ["record", "primitive", "float"] +} +public isolated function testRecordsWithFloatFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "float"} + ] + }`; + + Students student = { + name: "Liam", + age: 52.656 + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(student); + Students deserialize = check avro.fromAvro(encode); + test:assertEquals(student, deserialize); +} + +@test:Config { + groups: ["record", "primitive", "double"] +} +public isolated function testRecordsWithDoubleFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "double"} + ] + }`; + + Students student = { + name: "Liam", + age: 52.656 + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(student); + Students deserialize = check avro.fromAvro(encode); + test:assertEquals(student, deserialize); +} + +@test:Config { + groups: ["record", "primitive", "boolean"] +} +public isolated function testRecordsWithBooleanFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "under19", "type": "boolean"} + ] + }`; + + StudentRec student = { + name: "Liam", + under19: false + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(student); + StudentRec deserialize = check avro.fromAvro(encode); + test:assertEquals(student, deserialize); +} + +@test:Config { + groups: ["record", "union"] +} +public isolated function testOptionalValuesInRecords() returns error? { + string schema = string ` + { + "type": "record", + "name": "Lecturer5", + "fields": [ + { + "name": "name", + "type": [ + "null", + { + "type": "map", + "values": "int" + } + ] + }, + { + "name": "bytes", + "type": ["null", "bytes"] + }, + { + "name": "instructorClone", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }] + }, + { + "name": "instructors", + "type": ["null", "Instructor"] + } + ] + }`; + + Instructor instructor = { + name: "John", + student: { + name: "Alice", + subject: "Math" + } + }; + + Lecturer5 lecturer5 = { + name: { + "John": 1, + "Sam": 2, + "Liam": 3 + }, + bytes: "ss".toBytes(), + instructorClone: instructor.cloneReadOnly(), + instructors: instructor + }; + + Schema avro = check new (schema); + byte[] serialize = check avro.toAvro(lecturer5); + Lecturer5 deserialize = check avro.fromAvro(serialize); + test:assertEquals(deserialize, lecturer5); +} + +@test:Config { + groups: ["record", "union"] +} +public isolated function testOptionalMultipleFieldsInRecords() returns error? { + string schema = string ` + { + "type": "record", + "name": "Lecturer6", + "fields": [ + { + "name": "temporary", + "type": ["null", "boolean"] + }, + { + "name": "maps", + "type": [ + "null", + { + "type": "map", + "values": "int" + } + ] + }, + { + "name": "number", + "type": [ + "null", + { + "type": "enum", + "name": "Numbers", + "symbols": [ "ONE", "TWO", "THREE", "FOUR" ] + } + ] + }, + { + "name": "bytes", + "type": ["null", { + "type": "fixed", + "name": "FixedBytes", + "size": 2 + }] + }, + { + "name": "age", + "type": ["null", "long"] + }, + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "floatNumber", + "type": ["null", "float"] + }, + { + "name": "colors", + "type": ["null", { + "type": "array", + "items": { + "type": "enum", + "name": "ColorEnum", + "symbols": ["ONE", "TWO", "THREE"] + } + }] + }, + { + "name": "instructorClone", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + }] + }] + }, + { + "name": "instructors", + "type": ["null", "Instructor"] + } + ] + }`; + + Instructor instructor = { + name: "John", + student: { + name: "Alice", + subject: "Math" + } + }; + + Numbers number = ONE; + + Lecturer6 lecturer6 = { + temporary: false, + maps: { + "1": 100, + "2": 200 + }, + bytes: "ss".toBytes(), + age: 30, + number: number, + name: "Lecturer Name", + floatNumber: 123.45, + colors: [number, number, number], + instructorClone: instructor.cloneReadOnly(), + instructors: instructor + }; + + Schema avro = check new (schema); + byte[] serialize = check avro.toAvro(lecturer6); + Lecturer6 deserialize = check avro.fromAvro(serialize); + test:assertEquals(deserialize, lecturer6); +} diff --git a/ballerina/tests/test.bal b/ballerina/tests/test.bal index 3ad40ee..9494089 100644 --- a/ballerina/tests/test.bal +++ b/ballerina/tests/test.bal @@ -17,7 +17,9 @@ import ballerina/io; import ballerina/test; -@test:Config {} +@test:Config { + groups: ["enum"] +} public isolated function testEnums() returns error? { string schema = string ` { @@ -36,7 +38,7 @@ public isolated function testEnums() returns error? { } @test:Config { - groups: ["errors"] + groups: ["errors", "enum"] } public isolated function testEnumsWithString() returns error? { string schema = string ` @@ -50,373 +52,33 @@ public isolated function testEnumsWithString() returns error? { string number = "FIVE"; Schema avro = check new (schema); - byte[]|error encode = avro.toAvro(number); - test:assertTrue(encode is error); -} - -@test:Config {} -public isolated function testMaps() returns error? { - string schema = string ` - { - "type": "map", - "values" : "int", - "default": {} - }`; - - map colors = {"red": 0, "green": 1, "blue": 2}; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); - test:assertEquals(colors, deserialize); -} - -@test:Config {} -public isolated function testNestedRecords() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Lecturer", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "instructor", - "type": { - "name": "Instructor", - "type": "record", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "student", - "type": { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "subject", - "type": "string" - } - ] - } - } - ] - } - } - ] - }`; - - Lecturer lecturer = { - name: "John", - instructor: { - name: "Liam", - student: { - name: "Sam", - subject: "geology" - } - } - }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(lecturer); - Lecturer deserialize = check avro.fromAvro(serialize); - test:assertEquals(lecturer, deserialize); -} - -@test:Config {} -public isolated function testArraysInRecords() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Student", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "colors", "type": {"type": "array", "items": "string"}} - ] - }`; - - Color colors = { - name: "Red", - colors: ["maroon", "dark red", "light red"] - }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(colors); - Color deserialize = check avro.fromAvro(serialize); - test:assertEquals(colors, deserialize); + byte[]|Error encode = avro.toAvro(number); + test:assertTrue(encode is Error); } @test:Config { - groups: ["errors", "qwe"] -} -public isolated function testArraysInRecordsWithInvalidSchema() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Student", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "colors", "type": {"type": "array", "items": "string"}} - ] - }`; - - Color colors = { - name: "Red", - colors: ["maroon", "dark red", "light red"] - }; - - Schema avroProducer = check new (schema); - byte[] serialize = check avroProducer.toAvro(colors); - - string schema2 = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Student", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "colors", "type": {"type": "array", "items": "int"}} - ] - }`; - Schema avroConsumer = check new (schema2); - Color|Error deserialize = avroConsumer.fromAvro(serialize); - test:assertTrue(deserialize is Error); -} - -@test:Config {} -public isolated function testRecords() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Student", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "subject", "type": "string"} - ] - }`; - - Student student = { - name: "Liam", - subject: "geology" - }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(student); - Student deserialize = check avro.fromAvro(serialize); - test:assertEquals(student, deserialize); -} - -@test:Config {} -public isolated function testRecordsWithDifferentTypeOfFields() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Student", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "age", "type": "int"} - ] - }`; - - Person student = { - name: "Liam", - age: 52 - }; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(student); - Person deserialize = check avro.fromAvro(encode); - test:assertEquals(student, deserialize); -} - -@test:Config {} -public isolated function testRecordsWithUnionTypes() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Course", - "fields": [ - {"name": "name", "type": ["string", "null"]}, - {"name": "credits", "type": ["int", "null"]} - ] - }`; - - Course course = { - name: (), - credits: () - }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(course); - Course deserialize = check avro.fromAvro(serialize); - test:assertEquals(course, deserialize); -} - -@test:Config {} -public isolated function testArrays() returns error? { - string schema = string ` - { - "type": "array", - "name" : "StringArray", - "namespace": "data", - "items": "string" - }`; - - string[] colors = ["red", "green", "blue"]; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - string[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, colors); -} - -@test:Config {} -public isolated function testIntValue() returns error? { - string schema = string ` - { - "type": "int", - "name" : "intValue", - "namespace": "data" - }`; - - int value = 5; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - int deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + groups: ["fixed"] } - -@test:Config {} -public isolated function testFloatValue() returns error? { - string schema = string ` - { - "type": "float", - "name" : "floatValue", - "namespace": "data" - }`; - - float value = 5.5; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - float deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); -} - -@test:Config {} -public isolated function testDoubleValue() returns error? { +public isolated function testFixed() returns error? { string schema = string ` { - "type": "double", - "name" : "doubleValue", - "namespace": "data" + "type": "fixed", + "name": "name", + "size": 16 }`; - float value = 5.5595; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - float deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); -} - -@test:Config {} -public isolated function testLongValue() returns error? { - string schema = string ` - { - "type": "long", - "name" : "longValue", - "namespace": "data" - }`; + byte[] value = "u00ffffffffffffx".toBytes(); - int value = 555950000000000000; Schema avro = check new (schema); byte[] encode = check avro.toAvro(value); - int deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + byte[] deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, value); } @test:Config { - groups: ["primitive"] + groups: ["fixed"] } -public isolated function testStringValue() returns error? { - string schema = string ` - { - "type": "string", - "name" : "stringValue", - "namespace": "data" - }`; - - string value = "test"; - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - string deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); -} - -@test:Config {} -public isolated function testBoolean() returns error? { - string schema = string ` - { - "type": "boolean", - "name" : "booleanValue", - "namespace": "data" - }`; - - boolean value = true; - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - boolean deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); -} - -@test:Config {} -public isolated function testNullValues() returns error? { - string schema = string ` - { - "type": "null", - "name" : "nullValue", - "namespace": "data" - }`; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(()); - () deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, ()); -} - -@test:Config {} -public isolated function testNullValuesWithNonNullData() returns error? { - string schema = string ` - { - "type": "null", - "name" : "nullValue", - "namespace": "data" - }`; - - Schema avro = check new (schema); - byte[]|error encode = avro.toAvro("string"); - test:assertTrue(encode is error); -} - -@test:Config {} -public isolated function testFixed() returns error? { +public isolated function testFixedWithInvalidSize() returns error? { string schema = string ` { "type": "fixed", @@ -424,15 +86,16 @@ public isolated function testFixed() returns error? { "size": 16 }`; - byte[] value = "u00ffffffffffffx".toBytes(); + byte[] value = "u00".toBytes(); Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - byte[] deserialize = check avro.fromAvro(encode); - test:assertEquals(deserialize, value); + byte[]|Error encode = avro.toAvro(value); + test:assertTrue(encode is Error); } -@test:Config {} +@test:Config { + groups: ["record"] +} public function testDbSchemaWithRecords() returns error? { string schema = string ` { @@ -460,7 +123,9 @@ public function testDbSchemaWithRecords() returns error? { } -@test:Config {} +@test:Config { + groups: ["record"] +} public function testComplexDbSchema() returns error? { string jsonFileName = string `tests/resources/schema_1.json`; json result = check io:fileReadJson(jsonFileName); @@ -531,10 +196,11 @@ public function testComplexDbSchema() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(envelope); Envelope deserialize = check avro.fromAvro(serialize); - test:assertEquals(envelope, deserialize); + test:assertEquals(deserialize, envelope); } @test:Config { + groups: ["record"] } public function testComplexDbSchemaWithNestedRecords() returns error? { string jsonFileName = string `tests/resources/schema_2.json`; @@ -592,7 +258,7 @@ public function testComplexDbSchemaWithNestedRecords() returns error? { FirstName: "Jane", MiddleName: "K", Gender: "F", - Language: "Spanish", + Language: (), Discreet: true, Deceased: false, IsBanned: false, @@ -655,5 +321,5 @@ public function testComplexDbSchemaWithNestedRecords() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(envelope2); Envelope2 deserialize = check avro.fromAvro(serialize); - test:assertEquals(envelope2, deserialize); + test:assertEquals(deserialize, envelope2); } diff --git a/ballerina/tests/types.bal b/ballerina/tests/types.bal index ef73d18..ef61410 100644 --- a/ballerina/tests/types.bal +++ b/ballerina/tests/types.bal @@ -19,6 +19,21 @@ public type Student record { string subject; }; +public type Student1 record { + string name; + byte[] favorite_color; +}; + +type Students record { + string name; + float age; +}; + +type StudentRec record { + string name; + boolean under19; +}; + public type Person record { string name; int age; @@ -31,7 +46,7 @@ public type Course record { public type Instructor record { string? name; - Student student; + Student? student; }; public type Lecturer record { @@ -39,6 +54,65 @@ public type Lecturer record { Instructor instructor; }; +public type Lecturer1 readonly & record { + string? name; + Instructor & readonly instructor; +}; + +public type Lecturer2 record { + string? name; + int age; + Instructor & readonly instructor; +}; + +public type Lecturer3 readonly & record { + map & readonly name; + int age; + Instructor & readonly instructor; +}; + +public type Lecturer4 readonly & record { + map & readonly name; + byte[] byteData; + ByteRecord? instructor; +}; + +public type Lecturer5 record { + map? & readonly name; + byte[]? bytes; + Instructor? & readonly instructorClone; + Instructor? instructors; +}; + +public type Lecturer6 record { + boolean? temporary; + map? & readonly maps; + byte[]? bytes; + int? age; + string? name; + Numbers? number; + float? floatNumber; + string[]? colors; + Instructor? & readonly instructorClone; + Instructor? instructors; +}; + +public type ByteRecord readonly & record { + byte[] byteData; +}; + +type UnionRecord record { + string? name; + int? credits; + float value; + StudentRecord? student; +}; + +type StudentRecord record { + string? name; + string? subject; +}; + public type Color record { string? name; string[] colors; From 91773484f470901bb44b555b05224abd2d33d62c Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:49:25 +0530 Subject: [PATCH 14/47] Add test cases for map and array types --- ballerina/tests/array_tests.bal | 338 +++++++++++++++ ballerina/tests/map_tests.bal | 743 ++++++++++++++++++++++++++++++++ 2 files changed, 1081 insertions(+) create mode 100644 ballerina/tests/array_tests.bal create mode 100644 ballerina/tests/map_tests.bal diff --git a/ballerina/tests/array_tests.bal b/ballerina/tests/array_tests.bal new file mode 100644 index 0000000..bb90870 --- /dev/null +++ b/ballerina/tests/array_tests.bal @@ -0,0 +1,338 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; + +@test:Config { + groups: ["array", "int"] +} +public isolated function testIntArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "integers", + "namespace": "data", + "items": "int" + }`; + + int[] numbers = [22, 556, 78]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(numbers); + int[] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, numbers); +} + +@test:Config { + groups: ["array", "string"] +} +public isolated function testStringArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "stringArray", + "namespace": "data", + "items": "string" + }`; + + string[] colors = ["red", "green", "blue"]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + string[] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, colors); +} + +@test:Config { + groups: ["array", "string"] +} +public isolated function testEnumArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "enums", + "namespace": "data", + "items": { + "type": "enum", + "name": "Numbers", + "symbols": [ "ONE", "TWO", "THREE", "FOUR" ] + } + }`; + + Numbers[] colors = ["ONE", "TWO", "THREE"]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + Numbers[] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, colors); +} + +@test:Config { + groups: ["array", "string"] +} +public isolated function testArrayOfEnumArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "enums", + "namespace": "data", + "items": { + "type": "array", + "name" : "enumsValues", + "namespace": "data", + "items": { + "type": "enum", + "name": "Numbers", + "symbols": [ "ONE", "TWO", "THREE", "FOUR" ] + } + } + }`; + + Numbers[][] colors = [["ONE", "TWO", "THREE"], ["ONE", "TWO", "THREE"]]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + Numbers[][] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, colors); +} + +@test:Config { + groups: ["array", "float"] +} +public isolated function testFloatArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "floatArray", + "namespace": "data", + "items": "float" + }`; + + float[] numbers = [22.4, 556.84350, 78.0327]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(numbers); + float[] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, numbers); +} + +@test:Config { + groups: ["array", "double"] +} +public isolated function testDoubleArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "doubleArray", + "namespace": "data", + "items": "double" + }`; + + float[] numbers = [22.439475948, 556.843549485340, 78.032985693457]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(numbers); + float[] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, numbers); +} + +@test:Config { + groups: ["array", "long"] +} +public isolated function testLongArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "longArray", + "namespace": "data", + "items": "long" + }`; + + int[] numbers = [223432, 55423326, 7823423]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(numbers); + int[] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, numbers); +} + +@test:Config { + groups: ["array", "errors"] +} +public isolated function testInvalidDecimalArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "decimalArray", + "namespace": "data", + "items": "double" + }`; + + decimal[] numbers = [22.439475948, 556.843549485340, 78.032985693457]; + + Schema avro = check new (schema); + byte[]|Error encode = avro.toAvro(numbers); + test:assertTrue(encode is Error); +} + +@test:Config { + groups: ["array", "boolean"] +} +public isolated function testBooleanArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "booleanArray", + "namespace": "data", + "items": "boolean" + }`; + + boolean[] numbers = [true, true, false]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(numbers); + boolean[] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, numbers); +} + +@test:Config { + groups: ["array", "error", "anydata"] +} +public isolated function testArraysWithAnydata() returns error? { + string schema = string ` + { + "type": "array", + "name" : "floatArray", + "namespace": "data", + "items": "bytes" + }`; + + anydata numbers = ["22.4".toBytes(), "556.84350", 78.0327]; + Schema avro = check new (schema); + byte[]|Error encode = avro.toAvro(numbers); + test:assertTrue(encode is Error); +} + +@test:Config { + groups: ["array", "byte"] +} +public isolated function testArraysWithFixed() returns error? { + string schema = string ` + { + "type": "array", + "name" : "fixedArray", + "namespace": "data", + "items": { + "type": "fixed", + "name": "FixedBytes", + "size": 2 + } + }`; + + byte[][] numbers = ["22".toBytes(), "55".toBytes(), "78".toBytes()]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(numbers); + byte[][] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, numbers); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testRecordsInArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + }`; + + Student[] students = [{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(students); + Student[] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, students); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testRecordArraysInArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "array", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + } + }`; + + Student[][] students = [[{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }], [{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }]]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(students); + Student[][] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, students); +} diff --git a/ballerina/tests/map_tests.bal b/ballerina/tests/map_tests.bal new file mode 100644 index 0000000..26738ab --- /dev/null +++ b/ballerina/tests/map_tests.bal @@ -0,0 +1,743 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; + +@test:Config { + groups: ["maps", "bytes"] +} +public isolated function testMapsWithBytes() returns error? { + string schema = string ` + { + "type": "map", + "values" : "bytes", + "default": {} + }`; + + map colors = {"red": "0".toBytes(), "green": "1".toBytes(), "blue": "2".toBytes()}; + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "fixed"] +} +public isolated function testMapsWithFixed() returns error? { + string schema = string ` + { + "type": "map", + "values" : { + "type": "fixed", + "name": "name", + "size": 1 + }, + "default": {} + }`; + + map colors = {"red": "0".toBytes(), "green": "1".toBytes(), "blue": "2".toBytes()}; + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "fixed"] +} +public isolated function testMapsOfFixedMaps() returns error? { + string schema = string ` + { + "type": "map", + "values" : { + "type": "map", + "values" : { + "type": "fixed", + "name": "name", + "size": 1 + }, + "default": {} + }, + "default": {} + }`; + + map> colors = { + "red": {"r": "0".toBytes(), "g": "1".toBytes(), "b": "2".toBytes()}, + "green": {"r": "0".toBytes(), "g": "1".toBytes(), "b": "2".toBytes()}, + "blue": {"r": "0".toBytes(), "g": "1".toBytes(), "b": "2".toBytes()} + }; + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map> deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "record"] +} +public isolated function testMapsWithRecords() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} + }`; + + + + map instructors = { + "john": {name: "John", student: {name: "Alice", subject: "Math"}}, + "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, + "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(instructors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(instructors, deserialize); +} + +@test:Config { + groups: ["maps"] +} +public isolated function testMapsWithInt() returns error? { + string schema = string ` + { + "type": "map", + "values" : "int", + "default": {} + }`; + + map colors = {"red": 0, "green": 1, "blue": 2}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "enum"] +} +public isolated function testMapsWithEnum() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO", "THREE"] + }, + "default": {} + }`; + + map colors = {"red": "ONE", "green": "TWO", "blue": "THREE"}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "enum", "array"] +} +public isolated function testMapsWithEnumArrays() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "name" : "enums", + "namespace": "data", + "items": { + "type": "enum", + "name": "Numbers", + "symbols": [ "ONE", "TWO", "THREE", "FOUR" ] + } + }, + "default": {} + }`; + + map colors = {"red": ["ONE", "TWO", "THREE"], "green": ["ONE", "TWO", "THREE"], "blue": ["ONE", "TWO", "THREE"]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "float"] +} +public isolated function testMapsWithFloat() returns error? { + string schema = string ` + { + "type": "map", + "values": "float", + "default": {} + }`; + + map colors = {"red": 2.3453, "green": 435.563, "blue": 20347.23}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "double"] +} +public isolated function testMapsWithDouble() returns error? { + string schema = string ` + { + "type": "map", + "values": "double", + "default": {} + }`; + + map colors = {"red": 2.3453, "green": 435.563, "blue": 20347.23}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "double", "array"] +} +public isolated function testMapsWithDoubleArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "name" : "enums", + "namespace": "data", + "items": "double" + }, + "default": {} + }`; + + map colors = {"red": [2.3434253, 435.56433, 20347.22343], "green": [2.3452343, 435.56343, 20347.2423], "blue": [2.3453243, 435.56243, 20347.22343]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "long"] +} +public isolated function testMapsWithLong() returns error? { + string schema = string ` + { + "type": "map", + "values": "long", + "default": {} + }`; + + map colors = {"red": 2, "green": 435, "blue": 2034723}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "string"] +} +public isolated function testMapsWithStrings() returns error? { + string schema = string ` + { + "type": "map", + "values": "string", + "default": {} + }`; + + map colors = {"red": "2", "green": "435", "blue": "2034723"}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "boolean"] +} +public isolated function testMapsWithBoolean() returns error? { + string schema = string ` + { + "type": "map", + "values": "boolean", + "default": {} + }`; + + map colors = {"red": true, "green": false, "blue": false}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps"] +} +public isolated function testMapsWithMaps() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "map", + "values": "long" + }, + "default": {} + }`; + + map> colors = { + "red": {"r": 2, "g": 3, "b": 4}, + "green": {"r": 5, "g": 6, "b": 7}, + "blue": {"r": 8, "g": 9, "b": 10} + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map> deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "k"] +} +public isolated function testMapsWithNestedMaps() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "map", + "values": { + "type": "map", + "values": "int" + } + }, + "default": {} + }`; + + map>> colors = { + "red": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}}, + "green": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}}, + "blue": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}} + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map>> deserialize = check avro.fromAvro(encode); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["maps", "long"] +} +public isolated function testMapsWithLongArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "long" + }, + "default": {} + }`; + + map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["maps", "int", "az"] +} +public isolated function testMapsWithIntArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "int" + }, + "default": {} + }`; + + map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["maps", "float"] +} +public isolated function testMapsWithFloatArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "float" + }, + "default": {} + }`; + + // 207.234345 + map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["maps", "string"] +} +public isolated function testMapsWithStringArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "string" + }, + "default": {} + }`; + + map colors = {"red": ["252", "122", "41"], "green": ["235", "163", "23"], "blue": ["207", "123"]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["maps", "bytes"] +} +public isolated function testMapsWithBytesArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "bytes" + }, + "default": {} + }`; + + map colors = {"red": ["252".toBytes(), "122".toBytes(), "41".toBytes()], "green": ["235".toBytes(), "163".toBytes(), "23".toBytes()], "blue": ["207".toBytes(), "123".toBytes()]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["maps", "bytes"] +} +public isolated function testMapsWithFixedArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": { + "type": "fixed", + "name": "name", + "size": 3 + } + }, + "default": {} + }`; + + map colors = {"red": ["252".toBytes(), "122".toBytes(), "411".toBytes()], "green": ["235".toBytes(), "163".toBytes(), "213".toBytes()], "blue": ["207".toBytes(), "123".toBytes()]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["maps", "boolean"] +} +public isolated function testMapsWithBooleanArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "boolean" + }, + "default": {} + }`; + + map colors = {"red": [true, false, true], "green": [false, true, false], "blue": [true, false]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["maps", "record"] +} +public isolated function testMapsWithRecordArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + } + }, + "default": {} + }`; + + + + map instructors = { + "john": [{name: "John", student: {name: "Alice", subject: "Math"}}, {name: "John", student: {name: "Alice", subject: "Math"}}], + "doe": [{name: "Doe", student: {name: "Bob", subject: "Science"}}, {name: "Doe", student: {name: "Bob", subject: "Science"}}], + "jane": [{name: "Jane", student: {name: "Charlie", subject: "English"}}, {name: "Jane", student: {name: "Charlie", subject: "English"}}] + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(instructors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(instructors, deserialize); +} + +@test:Config { + groups: ["maps", "record"] +} +public isolated function testMapsWithNestedRecordMaps() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "map", + "values": { + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "instructor", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "subject", + "type": ["null", "string"] + } + ] + }] + } + ] + }] + } + ] + } + } + }`; + + + Lecturer lec = { + name: "John", + instructor: { + name: "Jane", + student: { + name: "Charlie", + subject: "English" + } + } + }; + + map> lecturers = { + "john": {"john": lec, "doe": lec}, + "doe": {"john": lec, "doe": lec}, + "jane": {"john": lec, "doe": lec} + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(lecturers); + map> deserialize = check avro.fromAvro(encode); + test:assertEquals(lecturers, deserialize); +} + + +@test:Config { + groups: ["maps", "record", "wx"] +} +public isolated function testMapsWithNestedRecordArrayMaps() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "map", + "values": { + "type": "array", + "items": { + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "instructor", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "subject", + "type": ["null", "string"] + } + ] + }] + } + ] + }] + } + ] + } + } + } + }`; + + + Lecturer[] lecs = [{name: "John", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}, + {name: "Doe", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}]; + + map> lecturers = { + "john": {"r": lecs, "g": lecs, "b": lecs}, + "doe": {"r": lecs, "g": lecs, "b": lecs}, + "jane": {"r": lecs, "g": lecs, "b": lecs} + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(lecturers); + map> deserialize = check avro.fromAvro(encode); + test:assertEquals(lecturers, deserialize); +} From 076a84e0854162006de8abc4ca6e121eaeacbce2 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 6 May 2024 07:49:33 +0530 Subject: [PATCH 15/47] Add test cases for bytes --- ballerina/tests/byte_tests.bal | 137 +++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 ballerina/tests/byte_tests.bal diff --git a/ballerina/tests/byte_tests.bal b/ballerina/tests/byte_tests.bal new file mode 100644 index 0000000..4d434da --- /dev/null +++ b/ballerina/tests/byte_tests.bal @@ -0,0 +1,137 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; + +@test:Config{ + groups: ["record", "bytes"] +} +public isolated function testRecordsWithBytes() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_color", "type": "bytes"} + ] + }`; + + Student1 student = { + name: "Liam", + favorite_color: "yellow".toBytes() + }; + + Schema avro = check new(schema); + byte[] encode = check avro.toAvro(student); + Student1 deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, student); +} + +@test:Config { + groups: ["array", "byte"] +} +public isolated function testArraysWithBytes() returns error? { + string schema = string ` + { + "type": "array", + "name" : "floatArray", + "namespace": "data", + "items": "bytes" + }`; + + byte[][] numbers = ["22.4".toBytes(), "556.84350".toBytes(), "78.0327".toBytes()]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(numbers); + byte[][] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, numbers); +} + +@test:Config { + groups: ["primitive", "bytes"] +} +public isolated function testBytes() returns error? { + string schema = string ` + { + "type": "bytes", + "name" : "byteValue", + "namespace": "data" + }`; + + byte[] value = "5".toBytes(); + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(value); + byte[] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, value); +} + +@test:Config { + groups: ["record", "maps", "bytes"] +} +public isolated function testNestedRecordsWithBytes() returns error? { + string schema = string ` + { + "type": "record", + "name": "Lecturer4", + "fields": [ + { + "name": "name", + "type": { + "type": "map", + "values": "int" + } + }, + { + "name": "byteData", + "type": "bytes" + }, + { + "name": "instructor", + "type": { + "type": "record", + "name": "ByteRecord", + "fields": [ + { + "name": "byteData", + "type": "bytes" + } + ] + } + } + ] + }`; + + Lecturer4 lecturer4 = { + name: { + "John": 1, + "Sam": 2, + "Liam": 3 + }, + byteData: "s".toBytes().cloneReadOnly(), + instructor: { + byteData: "ddd".toBytes().cloneReadOnly() + } + }; + + Schema avro = check new (schema); + byte[] serialize = check avro.toAvro(lecturer4); + Lecturer4 deserialize = check avro.fromAvro(serialize); + // deserialize.instructor.student.name = "Sam"; + test:assertEquals(deserialize, lecturer4); +} From 8e50f82e5169c1cfab1291e6dd035a07ae864b32 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 09:06:47 +0530 Subject: [PATCH 16/47] Move classes to separate packages --- .../visitor/DeserializeVisitor.java | 472 ++++++++++++++++++ .../visitor/IDeserializeVisitor.java | 12 +- .../lib/avro/serialize/UnionSerializer.java | 16 + .../visitor/ISerializeVisitor.java | 21 +- .../serialize/visitor/SerializeVisitor.java | 231 +++++++++ .../serialize/visitor/array/ArrayVisitor.java | 26 + .../visitor/array/ArrayVisitorFactory.java | 26 + .../visitor/array/EnumArrayVisitor.java | 24 + .../visitor/array/FixedArrayVisitor.java | 22 + .../visitor/array/IArrayVisitor.java | 9 + .../visitor/array/MapArrayVisitor.java | 28 ++ .../visitor/array/PrimitiveArrayVisitor.java | 72 +++ .../visitor/array/RecordArrayVisitor.java | 28 ++ .../visitor/array/UnionArrayVisitor.java | 37 ++ 14 files changed, 1003 insertions(+), 21 deletions(-) create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java rename native/src/main/java/io/ballerina/lib/avro/{ => deserialize}/visitor/IDeserializeVisitor.java (61%) create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java rename native/src/main/java/io/ballerina/lib/avro/{ => serialize}/visitor/ISerializeVisitor.java (56%) create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java new file mode 100644 index 0000000..e1d8cfd --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -0,0 +1,472 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize.visitor; + +import io.ballerina.lib.avro.deserialize.ArrayDeserializer; +import io.ballerina.lib.avro.deserialize.Deserializer; +import io.ballerina.lib.avro.deserialize.EnumDeserializer; +import io.ballerina.lib.avro.deserialize.FixedDeserializer; +import io.ballerina.lib.avro.deserialize.GenericDeserializer; +import io.ballerina.lib.avro.deserialize.MapDeserializer; +import io.ballerina.lib.avro.deserialize.RecordDeserializer; +import io.ballerina.lib.avro.deserialize.StringDeserializer; +import io.ballerina.lib.avro.deserialize.UnionDeserializer; +import io.ballerina.runtime.api.TypeTags; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.ArrayType; +import io.ballerina.runtime.api.types.Field; +import io.ballerina.runtime.api.types.IntersectionType; +import io.ballerina.runtime.api.types.MapType; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.ReferenceType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.utils.ValueUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static io.ballerina.lib.avro.Utils.ARRAY_TYPE; +import static io.ballerina.lib.avro.Utils.BOOLEAN_TYPE; +import static io.ballerina.lib.avro.Utils.FLOAT_TYPE; +import static io.ballerina.lib.avro.Utils.INTEGER_TYPE; +import static io.ballerina.lib.avro.Utils.RECORD_TYPE; +import static io.ballerina.lib.avro.Utils.REFERENCE_TYPE; +import static io.ballerina.lib.avro.Utils.STRING_TYPE; +import static io.ballerina.lib.avro.Utils.getMutableType; +import static io.ballerina.lib.avro.deserialize.visitor.UnionRecordUtils.visitUnionRecords; +import static io.ballerina.runtime.api.utils.StringUtils.fromString; + +public class DeserializeVisitor implements IDeserializeVisitor { + + public static Deserializer createDeserializer(Schema schema, Type type) { + return switch (schema.getElementType().getType()) { + case UNION -> + new UnionDeserializer(schema, type); + case ARRAY -> + new ArrayDeserializer(schema, type); + case ENUM -> + new EnumDeserializer(type); + case RECORD -> + new RecordDeserializer(schema, type); + case FIXED -> + new FixedDeserializer(type); + default -> + new GenericDeserializer(schema, type); + }; + } + + public BMap visit(RecordDeserializer recordDeserializer, GenericRecord rec) throws Exception { + Type originalType = recordDeserializer.getType(); + Type type = recordDeserializer.getType(); + Schema schema = recordDeserializer.getSchema(); + BMap avroRecord = createAvroRecord(type); + for (Schema.Field field : schema.getFields()) { + Object fieldData = rec.get(field.name()); + switch (field.schema().getType()) { + case MAP -> + processMapField(avroRecord, field, fieldData); + case ARRAY -> + processArrayField(avroRecord, field, fieldData); + case BYTES -> + processBytesField(avroRecord, field, fieldData); + case RECORD -> + processRecordField(avroRecord, field, fieldData); + case STRING -> + processStringField(avroRecord, field, fieldData); + case INT -> + avroRecord.put(fromString(field.name()), Long.parseLong(fieldData.toString())); + case FLOAT -> + avroRecord.put(fromString(field.name()), Double.parseDouble(fieldData.toString())); + case UNION -> + processUnionField(type, avroRecord, field, fieldData); + default -> + avroRecord.put(fromString(field.name()), fieldData); + } + } + + if (originalType.isReadOnly()) { + avroRecord.freezeDirect(); + } + return avroRecord; + } + + public BMap visit(MapDeserializer mapDeserializer, Map data) throws Exception { + BMap avroRecord = ValueCreator.createMapValue(); + Object[] keys = data.keySet().toArray(); + Schema schema = mapDeserializer.getSchema(); + Type type = mapDeserializer.getType(); + for (Object key : keys) { + Object value = data.get(key); + Schema.Type valueType = schema.getValueType().getType(); + switch (valueType) { + case ARRAY -> + processMapArray(avroRecord, schema, (MapType) type, key, (GenericData.Array) value); + case BYTES -> + avroRecord.put(fromString(key.toString()), + ValueCreator.createArrayValue(((ByteBuffer) value).array())); + case FIXED -> + avroRecord.put(fromString(key.toString()), + ValueCreator.createArrayValue(((GenericFixed) value).bytes())); + case ENUM, STRING -> + avroRecord.put(fromString(key.toString()), + fromString(value.toString())); + case RECORD -> + processMapRecord(avroRecord, schema, (MapType) type, key, (GenericRecord) value); + case FLOAT -> + avroRecord.put(fromString(key.toString()), Double.parseDouble(value.toString())); + case MAP -> + processMaps(avroRecord, schema, (MapType) type, key, (Map) value); + default -> + avroRecord.put(fromString(key.toString()), value); + } + } + return (BMap) ValueUtils.convert(avroRecord, type); + } + + public Object visit(GenericDeserializer genericDeserializer, GenericData.Array data) { + Schema schema = genericDeserializer.getSchema(); + switch (schema.getElementType().getType()) { + case STRING -> { + return visitStringArray(data); + } + case INT -> { + return visitIntArray(data); + } + case LONG -> { + return visitLongArray(data); + } + case FLOAT, DOUBLE -> { + return visitDoubleArray(data); + } + case BOOLEAN -> { + return visitBooleanArray(data); + } + default -> { + return visitBytesArray(data, genericDeserializer.getType()); + } + } + } + + public BArray visit(UnionDeserializer unionDeserializer, GenericData.Array data) throws Exception { + Type type = unionDeserializer.getType(); + Schema schema = unionDeserializer.getSchema(); + switch (((ArrayType) type).getElementType().getTag()) { + case STRING_TYPE -> { + return visitStringArray(data); + } + case FLOAT_TYPE -> { + return visitDoubleArray(data); + } + case BOOLEAN_TYPE -> { + return visitBooleanArray(data); + } + case INTEGER_TYPE -> { + return visitIntegerArray(data, schema); + } + case RECORD_TYPE -> { + RecordDeserializer recordDeserializer = new RecordDeserializer(schema.getElementType(), type); + return (BArray) recordDeserializer.visit(this, data); + } + case ARRAY_TYPE -> { + Object[] objects = new Object[data.size()]; + Type elementType = ((ArrayType) type).getElementType(); + ArrayDeserializer arrayDeserializer = new ArrayDeserializer(schema.getElementType(), elementType); + int index = 0; + for (Object currentData : data) { + Object deserializedObject = arrayDeserializer.visit(this, (GenericData.Array) currentData); + objects[index++] = deserializedObject; + } + return ValueCreator.createArrayValue(objects, (ArrayType) type); + + } + default -> { + return visitBytes(data); + } + } + } + + public BArray visit(RecordDeserializer recordDeserializer, GenericData.Array data) throws Exception { + List recordList = new ArrayList<>(); + Type type = recordDeserializer.getType(); + Schema schema = recordDeserializer.getSchema(); + switch (type.getTag()) { + case ARRAY_TYPE -> { + for (Object datum : data) { + Type fieldType = ((ArrayType) type).getElementType().getCachedReferredType(); + RecordDeserializer recordDes = new RecordDeserializer(schema.getElementType(), fieldType); + recordList.add(recordDes.visit(this, (GenericRecord) datum)); + } + } + case REFERENCE_TYPE -> { + for (Object datum : data) { + Type fieldType = ((ReferenceType) type).getReferredType(); + RecordDeserializer recordDes = new RecordDeserializer(schema.getElementType(), fieldType); + recordList.add(recordDes.visit(this, (GenericRecord) datum)); + } + } + } + assert type instanceof ArrayType; + return ValueCreator.createArrayValue(recordList.toArray(new Object[data.size()]), (ArrayType) type); + } + + private BMap createAvroRecord(Type type) { + if (type instanceof IntersectionType) { + type = getMutableType((IntersectionType) type); + } + return ValueCreator.createRecordValue((RecordType) type); + } + + private void processMapField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + Type mapType = extractMapType(avroRecord.getType()); + MapDeserializer mapDeserializer = new MapDeserializer(field.schema(), mapType); + Object fieldValue = mapDeserializer.visit(this, (Map) fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + private void processArrayField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + ArrayDeserializer arrayDes = new ArrayDeserializer(field.schema(), avroRecord.getType()); + Object fieldValue = arrayDes.visit(this, (GenericData.Array) fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + private void processBytesField(BMap avroRecord, Schema.Field field, Object fieldData) { + ByteBuffer byteBuffer = (ByteBuffer) fieldData; + Object fieldValue = ValueCreator.createArrayValue(byteBuffer.array()); + avroRecord.put(fromString(field.name()), fieldValue); + } + + private void processRecordField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + Type recType = extractRecordType((RecordType) avroRecord.getType()); + RecordDeserializer recordDes = new RecordDeserializer(field.schema(), recType); + Object fieldValue = recordDes.visit(this, (GenericRecord) fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + private void processStringField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + StringDeserializer stringDes = new StringDeserializer(); + Object fieldValue = stringDes.visit(this, fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + private void processUnionField(Type type, BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + visitUnionRecords(type, avroRecord, field, fieldData); + } + + private void processMaps(BMap avroRecord, Schema schema, + MapType type, Object key, Map value) throws Exception { + Schema fieldSchema = schema.getValueType(); + Type fieldType = type.getConstrainedType(); + MapDeserializer mapDes = new MapDeserializer(fieldSchema, fieldType); + Object fieldValue = mapDes.visit(this, value); + avroRecord.put(fromString(key.toString()), fieldValue); + } + + private void processMapRecord(BMap avroRecord, Schema schema, + MapType type, Object key, GenericRecord value) throws Exception { + Type fieldType = type.getConstrainedType().getCachedReferredType(); + RecordDeserializer recordDes = new RecordDeserializer(schema.getValueType(), fieldType); + Object fieldValue = recordDes.visit(this, value); + avroRecord.put(fromString(key.toString()), fieldValue); + } + + private void processMapArray(BMap avroRecord, Schema schema, + MapType type, Object key, GenericData.Array value) throws Exception { + Type fieldType = type.getConstrainedType(); + ArrayDeserializer arrayDeserializer = new ArrayDeserializer(schema.getValueType(), fieldType); + Object fieldValue = visit(arrayDeserializer, value); + avroRecord.put(fromString(key.toString()), fieldValue); + } + + public Object visit(ArrayDeserializer arrayDeserializer, GenericData.Array data) throws Exception { + Deserializer deserializer = createDeserializer(arrayDeserializer.getSchema(), arrayDeserializer.getType()); + return deserializer.visit(new DeserializeArrayVisitor(), data); + } + + public BArray visit(EnumDeserializer enumDeserializer, GenericData.Array data) { + Object[] enums = new Object[data.size()]; + for (int i = 0; i < data.size(); i++) { + enums[i] = visitString(data.get(i)); + } + return ValueCreator.createArrayValue(enums, (ArrayType) enumDeserializer.getType()); + } + + public BArray visit(FixedDeserializer fixedDeserializer, GenericData.Array data) { + Type type = fixedDeserializer.getType(); + List values = new ArrayList<>(); + for (Object datum : data) { + values.add(visitFixed(datum)); + } + return ValueCreator.createArrayValue(values.toArray(new BArray[data.size()]), (ArrayType) type); + } + + private static BArray visitIntegerArray(GenericData.Array data, Schema schema) { + for (Schema schemaInstance : schema.getElementType().getTypes()) { + if (schemaInstance.getType().equals(Schema.Type.INT)) { + return visitIntArray(data); + } + } + return visitLongArray(data); + } + + private BArray visitBytesArray(GenericData.Array data, Type type) { + List values = new ArrayList<>(); + for (Object datum : data) { + values.add(visitBytes(datum)); + } + return ValueCreator.createArrayValue(values.toArray(new BArray[data.size()]), (ArrayType) type); + } + + private static BArray visitBooleanArray(GenericData.Array data) { + boolean[] booleanArray = new boolean[data.size()]; + int index = 0; + for (Object datum : data) { + booleanArray[index++] = (boolean) datum; + } + return ValueCreator.createArrayValue(booleanArray); + } + + private BArray visitDoubleArray(GenericData.Array data) { + List doubleList = new ArrayList<>(); + for (Object datum : data) { + doubleList.add(visitDouble(datum)); + } + double[] doubleArray = doubleList.stream().mapToDouble(Double::doubleValue).toArray(); + return ValueCreator.createArrayValue(doubleArray); + } + + private static BArray visitLongArray(GenericData.Array data) { + List longList = new ArrayList<>(); + for (Object datum : data) { + longList.add((Long) datum); + } + long[] longArray = longList.stream().mapToLong(Long::longValue).toArray(); + return ValueCreator.createArrayValue(longArray); + } + + private static BArray visitIntArray(GenericData.Array data) { + List longList = new ArrayList<>(); + for (Object datum : data) { + longList.add(((Integer) datum).longValue()); + } + long[] longArray = longList.stream().mapToLong(Long::longValue).toArray(); + return ValueCreator.createArrayValue(longArray); + } + + private BArray visitStringArray(GenericData.Array data) { + BString[] stringArray = new BString[data.size()]; + for (int i = 0; i < data.size(); i++) { + stringArray[i] = visitString(data.get(i)); + } + return ValueCreator.createArrayValue(stringArray); + } + + + public double visitDouble(Object data) { + if (data instanceof Float) { + return Double.parseDouble(data.toString()); + } + return (double) data; + } + + public BArray visitBytes(Object data) { + return ValueCreator.createArrayValue(((ByteBuffer) data).array()); + } + + public BArray visitFixed(Object data) { + GenericData.Fixed fixed = (GenericData.Fixed) data; + return ValueCreator.createArrayValue(fixed.bytes()); + } + + public BString visitString(Object data) { + return fromString(data.toString()); + } + + public static Type extractMapType(Type type) { + Type mapType = type; + assert type instanceof RecordType; + for (Map.Entry entry : ((RecordType) type).getFields().entrySet()) { + Field fieldValue = entry.getValue(); + if (fieldValue != null) { + Type fieldType = fieldValue.getFieldType(); + switch (fieldType.getTag()) { + case TypeTags.MAP_TAG: + mapType = fieldType; + break; + case TypeTags.INTERSECTION_TAG: + Type referredType = getMutableType((IntersectionType) fieldType); + if (referredType.getTag() == TypeTags.MAP_TAG) { + mapType = referredType; + } + break; + default: + Type referType = TypeUtils.getReferredType(fieldType); + if (referType.getTag() == TypeTags.MAP_TAG) { + mapType = referType; + } + break; + } + } + } + return mapType; + } + + public static RecordType extractRecordType(RecordType type) { + Map fieldsMap = type.getFields(); + RecordType recType = type; + for (Map.Entry entry : fieldsMap.entrySet()) { + Field fieldValue = entry.getValue(); + if (fieldValue != null) { + Type fieldType = fieldValue.getFieldType(); + switch (fieldType.getTag()) { + case TypeTags.RECORD_TYPE_TAG: + recType = (RecordType) fieldType; + break; + case TypeTags.INTERSECTION_TAG: + Type getType = getMutableType((IntersectionType) fieldType); + if (getType.getTag() == TypeTags.RECORD_TYPE_TAG) { + recType = (RecordType) getType; + } + break; + default: + Type referredType = TypeUtils.getReferredType(fieldType); + if (referredType.getTag() == TypeTags.RECORD_TYPE_TAG) { + recType = (RecordType) referredType; + } + break; + } + } + } + return recType; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/visitor/IDeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/IDeserializeVisitor.java similarity index 61% rename from native/src/main/java/io/ballerina/lib/avro/visitor/IDeserializeVisitor.java rename to native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/IDeserializeVisitor.java index 153466a..7bac24c 100644 --- a/native/src/main/java/io/ballerina/lib/avro/visitor/IDeserializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/IDeserializeVisitor.java @@ -16,24 +16,14 @@ * under the License. */ -package io.ballerina.lib.avro.visitor; +package io.ballerina.lib.avro.deserialize.visitor; -import io.ballerina.runtime.api.types.Type; import io.ballerina.runtime.api.values.BArray; -import io.ballerina.runtime.api.values.BMap; import io.ballerina.runtime.api.values.BString; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; - -import java.util.Map; public interface IDeserializeVisitor { public double visitDouble(Object data); public BString visitString(Object data); - BMap visitMap(Map data, Type type, Schema schema) throws Exception; BArray visitFixed(Object data); - Object visitArray(Schema schema, GenericData.Array data, Type type) throws Exception; - BMap visitRecords(Type type, Schema schema, GenericRecord rec) throws Exception; } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java new file mode 100644 index 0000000..fbe4bc5 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java @@ -0,0 +1,16 @@ +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import org.apache.avro.Schema; + +public class UnionSerializer extends Serializer { + + public UnionSerializer(Schema schema) { + super(schema); + } + + @Override + public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { + return serializeVisitor.visitUnion(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/visitor/ISerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java similarity index 56% rename from native/src/main/java/io/ballerina/lib/avro/visitor/ISerializeVisitor.java rename to native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java index 557abc1..ae26d9a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/visitor/ISerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java @@ -16,23 +16,24 @@ * under the License. */ -package io.ballerina.lib.avro.visitor; +package io.ballerina.lib.avro.serialize.visitor; +import io.ballerina.lib.avro.serialize.ArraySerializer; +import io.ballerina.lib.avro.serialize.EnumSerializer; +import io.ballerina.lib.avro.serialize.FixedSerializer; +import io.ballerina.lib.avro.serialize.GenericSerializer; +import io.ballerina.lib.avro.serialize.RecordSerializer; import io.ballerina.runtime.api.values.BArray; import io.ballerina.runtime.api.values.BMap; -import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; -import java.util.Map; - public interface ISerializeVisitor { String visitString(Object data); - GenericRecord visitRecord(BMap data, Schema schema) throws Exception; - Map visitMap(BMap data, Schema schema) throws Exception; - GenericData.Array visitArray(BArray data, Schema schema) throws Exception; - Object visitBytes(Object data, Schema schema); - Object visitEnum(Object data, Schema schema); - GenericData.Fixed visitFixed(Object data, Schema schema); + GenericRecord visit(RecordSerializer recordSerializer, BMap data) throws Exception; + GenericData.Array visit(ArraySerializer arraySerializer, BArray data); + Object visit(EnumSerializer enumSerializer, Object data); + GenericData.Fixed visit(FixedSerializer fixedSerializer, Object data); + Object visit(GenericSerializer genericSerializer, Object data); } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java new file mode 100644 index 0000000..2e32b48 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor; + +import io.ballerina.lib.avro.serialize.ArraySerializer; +import io.ballerina.lib.avro.serialize.EnumSerializer; +import io.ballerina.lib.avro.serialize.FixedSerializer; +import io.ballerina.lib.avro.serialize.GenericSerializer; +import io.ballerina.lib.avro.serialize.MapSerializer; +import io.ballerina.lib.avro.serialize.RecordSerializer; +import io.ballerina.lib.avro.serialize.Serializer; +import io.ballerina.lib.avro.serialize.UnionSerializer; +import io.ballerina.lib.avro.serialize.visitor.array.ArrayVisitorFactory; +import io.ballerina.lib.avro.serialize.visitor.array.IArrayVisitor; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static io.ballerina.lib.avro.Utils.ARRAY_TYPE; +import static io.ballerina.lib.avro.Utils.FLOAT_TYPE; +import static io.ballerina.lib.avro.Utils.INTEGER_TYPE; +import static io.ballerina.lib.avro.Utils.MAP_TYPE; +import static io.ballerina.lib.avro.Utils.RECORD_TYPE; +import static io.ballerina.lib.avro.Utils.STRING_TYPE; + +public class SerializeVisitor implements ISerializeVisitor { + + public Serializer createSerializer(Schema schema) { + return switch (schema.getValueType().getType()) { + case INT, LONG, FLOAT, DOUBLE, BOOLEAN, STRING, BYTES -> + new GenericSerializer(schema.getValueType()); + case RECORD -> + new RecordSerializer(schema.getValueType()); + case MAP -> + new MapSerializer(schema.getValueType()); + case ARRAY -> + new ArraySerializer(schema.getValueType()); + case ENUM -> + new EnumSerializer(schema.getValueType()); + case FIXED -> + new FixedSerializer(schema.getValueType()); + default -> + throw new IllegalArgumentException("Unsupported schema type: " + schema.getValueType().getType()); + }; + } + + @Override + public String visitString(Object data) { + return ((BString) data).getValue(); + } + + @Override + public GenericRecord visit(RecordSerializer recordSerializer, BMap data) throws Exception { + GenericRecord genericRecord = new GenericData.Record(recordSerializer.getSchema()); + for (Schema.Field field : recordSerializer.getSchema().getFields()) { + Object fieldData = data.get(StringUtils.fromString(field.name())); + genericRecord.put(field.name(), serializeField(field.schema(), fieldData)); + } + return genericRecord; + } + + private Object serializeField(Schema schema, Object fieldData) throws Exception { + Schema.Type type = schema.getType(); + return switch (type) { + case RECORD -> + new RecordSerializer(schema).convert(this, fieldData); + case MAP -> + new MapSerializer(schema).convert(this, fieldData); + case ARRAY -> + new ArraySerializer(schema).convert(this, fieldData); + case ENUM -> + new EnumSerializer(schema).convert(this, fieldData); + case UNION -> + new UnionSerializer(schema).convert(this, fieldData); + default -> + new GenericSerializer(schema).convert(this, fieldData); + }; + } + + @Override + public Object visit(GenericSerializer genericSerializer, Object data) { + switch (genericSerializer.getSchema().getType()) { + case INT -> { + return ((Long) data).intValue(); + } + case FLOAT -> { + return ((Double) data).floatValue(); + } + case BYTES -> { + ByteBuffer byteBuffer = ByteBuffer.allocate(((BArray) data).getByteArray().length); + byteBuffer.put(((BArray) data).getByteArray()); + byteBuffer.position(0); + return byteBuffer; + } + case STRING -> { + return data.toString(); + } + default -> { + return data; + } + } + } + + public Map visit(MapSerializer mapSerializer, BMap data) throws Exception { + Map avroMap = new HashMap<>(); + Schema schema = mapSerializer.getSchema(); + if (schema.getType().equals(Schema.Type.UNION)) { + for (Schema fieldSchema: schema.getTypes()) { + if (fieldSchema.getType().equals(Schema.Type.MAP)) { + schema = fieldSchema; + } + } + } + for (Object value : data.getKeys()) { + Serializer serializer = createSerializer(schema); + avroMap.put(value.toString(), serializer.convert(this, data.get(value))); + } + return avroMap; + } + + @Override + public Object visit(EnumSerializer enumSerializer, Object data) { + return new GenericData.EnumSymbol(enumSerializer.getSchema(), data); + } + + @Override + public GenericData.Fixed visit(FixedSerializer fixedSerializer, Object data) { + return new GenericData.Fixed(fixedSerializer.getSchema(), ((BArray) data).getByteArray()); + } + + public GenericData.Array visit(ArraySerializer arraySerializer, BArray data) { + GenericData.Array array = new GenericData.Array<>(data.size(), arraySerializer.getSchema()); + IArrayVisitor visitor = ArrayVisitorFactory.createVisitor(arraySerializer.getSchema()); + return Objects.requireNonNull(visitor).visit(data, arraySerializer.getSchema(), array); + } + + public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Exception { + Schema fieldSchema = unionSerializer.getSchema(); + Type typeName = TypeUtils.getType(data); + switch (typeName.getTag()) { + case STRING_TYPE -> { + return fieldSchema.getTypes().stream() + .filter(type -> type.getType().equals(Schema.Type.ENUM)) + .findFirst() + .map(type -> visit(new EnumSerializer(type), data)) + .orElse(visit(new GenericSerializer(fieldSchema), data.toString())); + } + case ARRAY_TYPE -> { + for (Schema schema : fieldSchema.getTypes()) { + switch (schema.getType()) { + case BYTES -> { + return new GenericSerializer(schema).convert(this, data); + } + case FIXED -> { + return new FixedSerializer(schema).convert(this, data); + } + case ARRAY -> { + return new ArraySerializer(schema).convert(this, data); + } + } + } + return new ArraySerializer(fieldSchema).convert(this, data); + } + case MAP_TYPE -> { + return new MapSerializer(fieldSchema).convert(this, data); + } + case RECORD_TYPE -> { + return new RecordSerializer(getRecordSchema(Schema.Type.RECORD, fieldSchema.getTypes())) + .convert(this, data); + } + case INTEGER_TYPE -> { + return fieldSchema.getTypes().stream() + .filter(schema -> schema.getType().equals(Schema.Type.INT)) + .findFirst() + .map(schema -> new GenericSerializer(schema).convert(this, data)) + .orElse(data); + + } + case FLOAT_TYPE -> { + return fieldSchema.getTypes().stream() + .filter(schema -> schema.getType().equals(Schema.Type.FLOAT)) + .findFirst() + .map(schema -> new GenericSerializer(schema).convert(this, data)) + .orElse(data); + + } + default -> { + return data; + } + } + } + + public static Schema getRecordSchema(Schema.Type givenType, List schemas) { + for (Schema schema: schemas) { + if (schema.getType().equals(Schema.Type.UNION)) { + getRecordSchema(givenType, schema.getTypes()); + } else if (schema.getType().equals(givenType)) { + return schema; + } + } + return null; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java new file mode 100644 index 0000000..fd9afbe --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java @@ -0,0 +1,26 @@ +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.lib.avro.serialize.ArraySerializer; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Arrays; +import java.util.Objects; + +public class ArrayVisitor implements IArrayVisitor { + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(value -> { + try { + array.add(new SerializeVisitor().visit(new ArraySerializer(schema.getElementType()), + (BArray) value)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java new file mode 100644 index 0000000..36a9634 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java @@ -0,0 +1,26 @@ +package io.ballerina.lib.avro.serialize.visitor.array; + +import org.apache.avro.Schema; + +public class ArrayVisitorFactory { + public static IArrayVisitor createVisitor(Schema schema) { + switch (schema.getElementType().getType()) { + case NULL: + return null; + case ARRAY: + return new ArrayVisitor(); + case ENUM: + return new EnumArrayVisitor(); + case UNION: + return new UnionArrayVisitor(); + case FIXED: + return new FixedArrayVisitor(); + case RECORD: + return new RecordArrayVisitor(); + case MAP: + return new MapArrayVisitor(); + default: + return new PrimitiveArrayVisitor(); + } + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java new file mode 100644 index 0000000..5757ac4 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java @@ -0,0 +1,24 @@ +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Arrays; +import java.util.Objects; + +public class EnumArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Arrays.stream((data.getValues() == null) ? data.getStringArray() : data.getValues()) + .filter(Objects::nonNull) + .forEach(value -> { + try { + array.add(new GenericData.EnumSymbol(schema.getElementType(), value)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java new file mode 100644 index 0000000..9a401dc --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java @@ -0,0 +1,22 @@ +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericFixed; + +import java.util.Arrays; +import java.util.Objects; + +public class FixedArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(bytes -> { + GenericFixed genericFixed = new GenericData.Fixed(schema, ((BArray) bytes).getByteArray()); + array.add(genericFixed); + }); + return array; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java new file mode 100644 index 0000000..2645378 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java @@ -0,0 +1,9 @@ +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public interface IArrayVisitor { + GenericData.Array visit(BArray data, Schema schema, GenericData.Array array); +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java new file mode 100644 index 0000000..1674e96 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java @@ -0,0 +1,28 @@ +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.lib.avro.serialize.MapSerializer; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Arrays; +import java.util.Objects; + +public class MapArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(record -> { + try { + array.add(new SerializeVisitor().visit(new MapSerializer(schema.getElementType()), + (BMap) record)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java new file mode 100644 index 0000000..399ab75 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java @@ -0,0 +1,72 @@ +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Objects; + +public class PrimitiveArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Schema.Type type = schema.getType().equals(Schema.Type.ARRAY) + ? schema.getElementType().getType() + : schema.getType(); + + switch (type) { + case STRING -> { + array.addAll(Arrays.asList(data.getStringArray())); + return array; + } + case INT -> { + for (long obj: data.getIntArray()) { + array.add(((Long) obj).intValue()); + } + return array; + } + case LONG -> { + for (Object obj: data.getIntArray()) { + array.add(obj); + } + return array; + } + case FLOAT -> { + for (Double obj: data.getFloatArray()) { + array.add(obj.floatValue()); + } + return array; + } + case DOUBLE -> { + for (Object obj: data.getFloatArray()) { + array.add(obj); + } + return array; + } + case BOOLEAN -> { + for (Object obj: data.getBooleanArray()) { + array.add(obj); + } + return array; + } + default -> { + return visitBytes(data, array); + } + } + } + + + public static GenericData.Array visitBytes(BArray data, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(bytes -> { + ByteBuffer byteBuffer = ByteBuffer.allocate(((BArray) bytes).getByteArray().length); + byteBuffer.put(((BArray) bytes).getByteArray()); + byteBuffer.position(0); + array.add(byteBuffer); + }); + return array; + } +} + diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java new file mode 100644 index 0000000..93ff3d0 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java @@ -0,0 +1,28 @@ +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.lib.avro.serialize.RecordSerializer; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Arrays; +import java.util.Objects; + +public class RecordArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(record -> { + try { + array.add(new SerializeVisitor() + .visit(new RecordSerializer(schema.getElementType()), (BMap) record)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java new file mode 100644 index 0000000..65b1280 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java @@ -0,0 +1,37 @@ +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Map; + +import static java.util.Map.entry; + +public class UnionArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Map visitorMap = Map.ofEntries( + entry(Schema.Type.ARRAY, new ArrayVisitor()), + entry(Schema.Type.MAP, new MapArrayVisitor()), + entry(Schema.Type.RECORD, new RecordArrayVisitor()), + entry(Schema.Type.FIXED, new FixedArrayVisitor()), + entry(Schema.Type.BOOLEAN, new PrimitiveArrayVisitor()), + entry(Schema.Type.STRING, new PrimitiveArrayVisitor()), + entry(Schema.Type.INT, new PrimitiveArrayVisitor()), + entry(Schema.Type.LONG, new PrimitiveArrayVisitor()), + entry(Schema.Type.DOUBLE, new PrimitiveArrayVisitor()), + entry(Schema.Type.BYTES, new PrimitiveArrayVisitor()), + entry(Schema.Type.FLOAT, new PrimitiveArrayVisitor()) + ); + + Schema elementType = schema.getElementType(); + for (Schema schema1 : elementType.getTypes()) { + IArrayVisitor visitor = visitorMap.get(schema1.getType()); + if (visitor != null) { + return visitor.visit(data, schema1, array); + } + } + return null; + } +} From 17fec3f7119e68081671740558a048d96096bc44 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 09:07:17 +0530 Subject: [PATCH 17/47] Update APIs of the deserializer class --- .../avro/deserialize/ArrayDeserializer.java | 11 ++++++--- .../avro/deserialize/ByteDeserializer.java | 11 ++++++--- .../avro/deserialize/DeserializeFactory.java | 6 ++--- .../lib/avro/deserialize/Deserializer.java | 14 ++++++----- .../avro/deserialize/DoubleDeserializer.java | 11 ++++++--- .../avro/deserialize/EnumDeserializer.java | 23 +++++++++++++++++++ .../avro/deserialize/FixedDeserializer.java | 17 +++++++++++--- .../avro/deserialize/GenericDeserializer.java | 15 ++++++++++-- .../lib/avro/deserialize/MapDeserializer.java | 20 ++++++++++++---- .../avro/deserialize/NullDeserializer.java | 11 ++++++--- .../avro/deserialize/RecordDeserializer.java | 16 ++++++++++--- .../avro/deserialize/StringDeserializer.java | 15 +++++++++--- .../avro/deserialize/UnionDeserializer.java | 23 +++++++++++++++++++ 13 files changed, 156 insertions(+), 37 deletions(-) create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java index a84d49e..fcfb483 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java @@ -18,7 +18,7 @@ package io.ballerina.lib.avro.deserialize; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; import io.ballerina.runtime.api.types.Type; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; @@ -30,7 +30,12 @@ public ArrayDeserializer(Schema schema, Type type) { } @Override - public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception { - return visitor.visitArray(getSchema(), (GenericData.Array) data, getType()); + public Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception { + return visitor.visit(this, (GenericData.Array) data); + } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java index 7825dfe..508bd16 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java @@ -1,11 +1,16 @@ package io.ballerina.lib.avro.deserialize; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; -import org.apache.avro.Schema; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import org.apache.avro.generic.GenericData; public class ByteDeserializer extends Deserializer { @Override - public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception { + public Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception { + return visitor.visitBytes(data); + } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visitBytes(data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java index a98f81d..b8c1749 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java @@ -29,11 +29,11 @@ public static Deserializer generateDeserializer(Schema schema, Type type) { case FLOAT, DOUBLE -> new DoubleDeserializer(); case STRING, ENUM -> new StringDeserializer(); case ARRAY -> new ArrayDeserializer(schema, type); - case FIXED -> new FixedDeserializer(); - case MAP -> new MapDeserializer(type); + case FIXED -> new FixedDeserializer(type); + case MAP -> new MapDeserializer(schema, type); case RECORD -> new RecordDeserializer(schema, type); case BYTES -> new ByteDeserializer(); - default -> new GenericDeserializer(); + default -> new GenericDeserializer(schema, type); }; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java index eede4b9..d3e384b 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java @@ -18,10 +18,11 @@ package io.ballerina.lib.avro.deserialize; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; import io.ballerina.runtime.api.types.Type; import io.ballerina.runtime.api.utils.TypeUtils; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; public abstract class Deserializer { @@ -43,13 +44,14 @@ public Deserializer(Schema schema, Type type) { this.type = TypeUtils.getReferredType(type); } - protected Schema getSchema() { - return schema; + public Schema getSchema() { + return new Schema.Parser().parse(schema.toString()); } - protected Type getType() { - return type; + public Type getType() { + return TypeUtils.getReferredType(type); } - public abstract Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception; + public abstract Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception; + public abstract Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception; } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java index 9f1482e..9a9bac9 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java @@ -18,13 +18,18 @@ package io.ballerina.lib.avro.deserialize; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; -import org.apache.avro.Schema; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import org.apache.avro.generic.GenericData; public class DoubleDeserializer extends Deserializer { @Override - public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) { + public Object fromAvro(DeserializeVisitor visitor, Object data) { return visitor.visitDouble(data); } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return null; + } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java new file mode 100644 index 0000000..60664c3 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java @@ -0,0 +1,23 @@ +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.generic.GenericData; + +public class EnumDeserializer extends Deserializer { + + public EnumDeserializer(Type type) { + super(type); + } + + @Override + public Object fromAvro(DeserializeVisitor visitor, Object data) { + return visitor.visit(this, (GenericData.Array) data); + } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } + +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java index 95eba4c..cd358eb 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java @@ -18,13 +18,24 @@ package io.ballerina.lib.avro.deserialize; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; -import org.apache.avro.Schema; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.generic.GenericData; public class FixedDeserializer extends Deserializer { + public FixedDeserializer(Type type) { + super(type); + } + @Override - public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) { + public Object fromAvro(DeserializeVisitor visitor, Object data) { return visitor.visitFixed(data); } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } + } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java index d2242b0..f4ca25d 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java @@ -18,13 +18,24 @@ package io.ballerina.lib.avro.deserialize; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; public class GenericDeserializer extends Deserializer { + public GenericDeserializer(Schema schema, Type type) { + super(schema, type); + } + @Override - public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) { + public Object fromAvro(DeserializeVisitor visitor, Object data) { return data; } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java index f04df55..e259a6c 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java @@ -18,20 +18,30 @@ package io.ballerina.lib.avro.deserialize; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; import io.ballerina.runtime.api.types.Type; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; import java.util.Map; public class MapDeserializer extends Deserializer { - public MapDeserializer(Type type) { - super(type); + public MapDeserializer(Schema schema, Type type) { + super(schema, type); } @Override - public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception { - return visitor.visitMap((Map) data, getType(), schema); + public Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception { + return visitor.visit(this, (Map) data); + } + + public Object visit(DeserializeVisitor visitor, Map data) throws Exception { + return visitor.visit(this, data); + } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return null; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java index 91f0801..8dbafd4 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java @@ -18,16 +18,21 @@ package io.ballerina.lib.avro.deserialize; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; -import org.apache.avro.Schema; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import org.apache.avro.generic.GenericData; public class NullDeserializer extends Deserializer { @Override - public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception { + public Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception { if (data != null) { throw new Exception("The value does not match with the null schema"); } return null; } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return null; + } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java index 17bf236..0cd21f6 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java @@ -18,9 +18,10 @@ package io.ballerina.lib.avro.deserialize; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; import io.ballerina.runtime.api.types.Type; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; public class RecordDeserializer extends Deserializer { @@ -30,7 +31,16 @@ public RecordDeserializer(Schema schema, Type type) { } @Override - public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) throws Exception { - return visitor.visitRecords(getType(), getSchema(), (GenericRecord) data); + public Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception { + return visitor.visit(this, (GenericRecord) data); + } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } + + public Object visit(DeserializeVisitor visitor, GenericRecord data) throws Exception { + return visitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java index b661c85..873f2f3 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java @@ -18,13 +18,22 @@ package io.ballerina.lib.avro.deserialize; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; -import org.apache.avro.Schema; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import org.apache.avro.generic.GenericData; public class StringDeserializer extends Deserializer { @Override - public Object fromAvroMessage(DeserializeVisitor visitor, Object data, Schema schema) { + public Object fromAvro(DeserializeVisitor visitor, Object data) { + return visitor.visitString(data); + } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return null; + } + + public Object visit(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visitString(data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java new file mode 100644 index 0000000..51901e0 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java @@ -0,0 +1,23 @@ +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public class UnionDeserializer extends Deserializer { + + public UnionDeserializer(Schema schema, Type type) { + super(schema, type); + } + + @Override + public Object fromAvro(DeserializeVisitor visitor, Object data) { + return null; + } + + @Override + public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } +} From 83f3ceaf4db4eaf8a2567fd3aba3f3eb6dec9b67 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 09:08:37 +0530 Subject: [PATCH 18/47] Update API of the Serializer classes --- .../lib/avro/serialize/ArraySerializer.java | 6 +- .../lib/avro/serialize/ByteSerializer.java | 5 +- .../lib/avro/serialize/EnumSerializer.java | 6 +- .../lib/avro/serialize/FixedSerializer.java | 6 +- .../lib/avro/serialize/GenericSerializer.java | 11 +- .../lib/avro/serialize/MapSerializer.java | 6 +- .../lib/avro/serialize/NullSerializer.java | 4 +- .../lib/avro/serialize/RecordSerializer.java | 6 +- .../lib/avro/serialize/Serializer.java | 44 +- .../lib/avro/serialize/StringSerializer.java | 4 +- .../lib/avro/visitor/DeserializeVisitor.java | 411 ------------------ .../lib/avro/visitor/SerializeVisitor.java | 348 --------------- 12 files changed, 50 insertions(+), 807 deletions(-) delete mode 100644 native/src/main/java/io/ballerina/lib/avro/visitor/DeserializeVisitor.java delete mode 100644 native/src/main/java/io/ballerina/lib/avro/visitor/SerializeVisitor.java diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java index f117529..7b51923 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java @@ -18,7 +18,7 @@ package io.ballerina.lib.avro.serialize; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import io.ballerina.runtime.api.values.BArray; import org.apache.avro.Schema; @@ -29,7 +29,7 @@ public ArraySerializer(Schema schema) { } @Override - public Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception { - return serializeVisitor.visitArray((BArray) data, getSchema()); + public Object convert(SerializeVisitor serializeVisitor, Object data) { + return serializeVisitor.visit(this, (BArray) data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java index 7ea67f5..705d649 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java @@ -1,13 +1,14 @@ package io.ballerina.lib.avro.serialize; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import io.ballerina.runtime.api.values.BArray; import java.nio.ByteBuffer; public class ByteSerializer extends Serializer { + @Override - public Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception { + public Object convert(SerializeVisitor serializeVisitor, Object data) { return ByteBuffer.wrap(((BArray) data).getByteArray()); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java index d14bd3c..9f50b5a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java @@ -18,7 +18,7 @@ package io.ballerina.lib.avro.serialize; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import org.apache.avro.Schema; public class EnumSerializer extends Serializer { @@ -28,7 +28,7 @@ public EnumSerializer(Schema schema) { } @Override - public Object generateMessage(SerializeVisitor serializeVisitor, Object data) { - return serializeVisitor.visitEnum(data, getSchema()); + public Object convert(SerializeVisitor serializeVisitor, Object data) { + return serializeVisitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java index ad4af55..68b9317 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java @@ -18,7 +18,7 @@ package io.ballerina.lib.avro.serialize; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import org.apache.avro.Schema; public class FixedSerializer extends Serializer { @@ -28,7 +28,7 @@ public FixedSerializer(Schema schema) { } @Override - public Object generateMessage(SerializeVisitor serializeVisitor, Object data) { - return serializeVisitor.visitFixed(data, getSchema()); + public Object convert(SerializeVisitor serializeVisitor, Object data) { + return serializeVisitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java index 1f2fe04..d3fc21d 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java @@ -18,12 +18,17 @@ package io.ballerina.lib.avro.serialize; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import org.apache.avro.Schema; public class GenericSerializer extends Serializer { + public GenericSerializer(Schema schema) { + super(schema); + } + @Override - public Object generateMessage(SerializeVisitor serializeVisitor, Object data) { - return data; + public Object convert(SerializeVisitor serializeVisitor, Object data) { + return serializeVisitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java index d2da813..b7be25b 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java @@ -18,7 +18,7 @@ package io.ballerina.lib.avro.serialize; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import io.ballerina.runtime.api.values.BMap; import org.apache.avro.Schema; @@ -29,7 +29,7 @@ public MapSerializer(Schema schema) { } @Override - public Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception { - return serializeVisitor.visitMap((BMap) data, getSchema()); + public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { + return serializeVisitor.visit(this, (BMap) data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java index d6b2345..e6281ef 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java @@ -18,12 +18,12 @@ package io.ballerina.lib.avro.serialize; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; public class NullSerializer extends Serializer { @Override - public Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception { + public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { if (data != null) { throw new Exception("The value does not match with the null schema"); } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java index 63e5825..bb9b7fb 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java @@ -18,7 +18,7 @@ package io.ballerina.lib.avro.serialize; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import io.ballerina.runtime.api.values.BMap; import org.apache.avro.Schema; @@ -29,7 +29,7 @@ public RecordSerializer(Schema schema) { } @Override - public Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception { - return serializeVisitor.visitRecord((BMap) data, getSchema()); + public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { + return serializeVisitor.visit(this, (BMap) data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java index 45a788b..309945a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java @@ -1,41 +1,37 @@ -/* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package io.ballerina.lib.avro.serialize; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.TypeUtils; import org.apache.avro.Schema; public abstract class Serializer { - private final Schema schema; + private final String schema; + private final Type type; public Serializer() { + this.type = null; this.schema = null; } public Serializer(Schema schema) { - this.schema = new Schema.Parser().parse(schema.toString()); + this.type = null; + this.schema = schema.toString(); + } + + public Serializer(Schema schema, Type type) { + this.type = TypeUtils.getImpliedType(type); + this.schema = schema.toString(); + } + + public Schema getSchema() { + return new Schema.Parser().parse(schema); } - protected Schema getSchema() { - return schema; + public Type getType() { + return TypeUtils.getImpliedType(type); } - public abstract Object generateMessage(SerializeVisitor serializeVisitor, Object data) throws Exception; + public abstract Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception; } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java index 035e42c..ade3fb6 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java @@ -18,7 +18,7 @@ package io.ballerina.lib.avro.serialize; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import org.apache.avro.Schema; public class StringSerializer extends Serializer { @@ -28,7 +28,7 @@ public StringSerializer(Schema schema) { } @Override - public Object generateMessage(SerializeVisitor serializeVisitor, Object data) { + public Object convert(SerializeVisitor serializeVisitor, Object data) { return serializeVisitor.visitString(data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/visitor/DeserializeVisitor.java deleted file mode 100644 index a6c63ce..0000000 --- a/native/src/main/java/io/ballerina/lib/avro/visitor/DeserializeVisitor.java +++ /dev/null @@ -1,411 +0,0 @@ -/* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.ballerina.lib.avro.visitor; - -import io.ballerina.runtime.api.creators.ValueCreator; -import io.ballerina.runtime.api.types.ArrayType; -import io.ballerina.runtime.api.types.Field; -import io.ballerina.runtime.api.types.IntersectionType; -import io.ballerina.runtime.api.types.MapType; -import io.ballerina.runtime.api.types.RecordType; -import io.ballerina.runtime.api.types.Type; -import io.ballerina.runtime.api.utils.StringUtils; -import io.ballerina.runtime.api.utils.TypeUtils; -import io.ballerina.runtime.api.values.BArray; -import io.ballerina.runtime.api.values.BMap; -import io.ballerina.runtime.api.values.BString; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericEnumSymbol; -import org.apache.avro.generic.GenericFixed; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.util.Utf8; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static io.ballerina.lib.avro.Utils.getMutableType; - -public class DeserializeVisitor implements IDeserializeVisitor { - - public double visitDouble(Object data) { - if (data instanceof Float) { - return Double.parseDouble(data.toString()); - } - return (double) data; - } - - public BArray visitBytes(Object data) { - return ValueCreator.createArrayValue(((ByteBuffer) data).array()); - } - - public BArray visitFixed(Object data) { - GenericData.Fixed fixed = (GenericData.Fixed) data; - return ValueCreator.createArrayValue(fixed.bytes()); - } - - public BString visitString(Object data) { - return StringUtils.fromString(data.toString()); - } - - @SuppressWarnings({"unchecked", "deprecation"}) - public BMap visitMap(Map data, Type type, Schema schema) throws Exception { - assert type instanceof MapType; - BMap avroRecord = ValueCreator.createMapValue(type); - Object[] keys = data.keySet().toArray(); - for (Object key : keys) { - Object value = data.get(key); - Schema.Type valueType = schema.getValueType().getType(); - switch (valueType) { - case ARRAY -> - avroRecord.put(StringUtils.fromString(key.toString()), visitArray(schema.getValueType(), - (GenericData.Array) value, ((MapType) type).getConstrainedType())); - case BYTES -> - avroRecord.put(StringUtils.fromString(key.toString()), - ValueCreator.createArrayValue(((ByteBuffer) value).array())); - case FIXED -> - avroRecord.put(StringUtils.fromString(key.toString()), - ValueCreator.createArrayValue(((GenericFixed) value).bytes())); - case RECORD -> - avroRecord.put(StringUtils.fromString(key.toString()), - visitRecords(((MapType) type).getConstrainedType().getCachedReferredType(), - schema.getValueType(), (GenericRecord) value)); - case ENUM, STRING -> - avroRecord.put(StringUtils.fromString(key.toString()), - StringUtils.fromString(value.toString())); - case FLOAT -> - avroRecord.put(StringUtils.fromString(key.toString()), Double.parseDouble(value.toString())); - case MAP -> - avroRecord.put(StringUtils.fromString(key.toString()), - visitMap((Map) value, - ((MapType) type).getConstrainedType(), schema.getValueType())); - default -> - avroRecord.put(StringUtils.fromString(key.toString()), value); - } - } - return avroRecord; - } - - public Object visitArray(Schema schema, GenericData.Array data, Type type) throws Exception { - switch (schema.getElementType().getType()) { - case ARRAY -> { - Object[] objects = new Object[data.size()]; - Type arrayType = ((ArrayType) type).getElementType(); - for (int i = 0; i < data.size(); i++) { - objects[i] = visitArray(schema.getElementType(), - (GenericData.Array) data.get(i), arrayType); - } - return ValueCreator.createArrayValue(objects, (ArrayType) type); - } - case STRING -> { - return visitStringArray(data); - } - case ENUM -> { - Object[] enums = new Object[data.size()]; - for (int i = 0; i < data.size(); i++) { - enums[i] = visitString(data.get(i)); - } - return ValueCreator.createArrayValue(enums, (ArrayType) type); - } - case INT -> { - return visitIntArray(data); - } - case LONG -> { - return visitLongArray(data); - } - case FLOAT, DOUBLE -> { - return visitDoubleArray(data); - } - case BOOLEAN -> { - return visitBooleanArray(data); - } - case RECORD -> { - return visitRecordArray(schema, data, type); - } - case FIXED -> { - assert type instanceof ArrayType; - return visitFixedArray(data, (ArrayType) type); - } - default -> { - assert type instanceof ArrayType; - return visitBytesArray(data, (ArrayType) type); - } - } - } - - private BArray visitBytesArray(GenericData.Array data, ArrayType type) { - List values = new ArrayList<>(); - for (Object datum : data) { - values.add(visitBytes(datum)); - } - return ValueCreator.createArrayValue(values.toArray(new BArray[data.size()]), type); - } - - private BArray visitFixedArray(GenericData.Array data, ArrayType type) { - List values = new ArrayList<>(); - for (Object datum : data) { - values.add(visitFixed(datum)); - } - return ValueCreator.createArrayValue(values.toArray(new BArray[data.size()]), type); - } - - private BArray visitRecordArray(Schema schema, GenericData.Array data, Type type) throws Exception { - List recordList = new ArrayList<>(); - if (type instanceof ArrayType arrayType) { - for (Object datum : data) { - recordList.add(visitRecords(arrayType.getElementType().getCachedReferredType(), - schema.getElementType(), (GenericRecord) datum)); - } - } - assert type instanceof ArrayType; - return ValueCreator.createArrayValue(recordList.toArray(new Object[data.size()]), (ArrayType) type); - - } - - private static BArray visitBooleanArray(GenericData.Array data) { - boolean[] booleanArray = new boolean[data.size()]; - int index = 0; - for (Object datum : data) { - booleanArray[index++] = (boolean) datum; - } - return ValueCreator.createArrayValue(booleanArray); - } - - private BArray visitDoubleArray(GenericData.Array data) { - List doubleList = new ArrayList<>(); - for (Object datum : data) { - doubleList.add(visitDouble(datum)); - } - double[] doubleArray = doubleList.stream().mapToDouble(Double::doubleValue).toArray(); - return ValueCreator.createArrayValue(doubleArray); - } - - private static BArray visitLongArray(GenericData.Array data) { - List longList = new ArrayList<>(); - for (Object datum : data) { - longList.add((Long) datum); - } - long[] longArray = longList.stream().mapToLong(Long::longValue).toArray(); - return ValueCreator.createArrayValue(longArray); - } - - private static BArray visitIntArray(GenericData.Array data) { - List longList = new ArrayList<>(); - for (Object datum : data) { - longList.add(((Integer) datum).longValue()); - } - long[] longArray = longList.stream().mapToLong(Long::longValue).toArray(); - return ValueCreator.createArrayValue(longArray); - } - - private BArray visitStringArray(GenericData.Array data) { - BString[] stringArray = new BString[data.size()]; - for (int i = 0; i < data.size(); i++) { - stringArray[i] = visitString(data.get(i)); - } - return ValueCreator.createArrayValue(stringArray); - } - - @SuppressWarnings("unchecked") - public BMap visitRecords(Type type, Schema schema, GenericRecord rec) throws Exception { - BMap avroRecord; - Type originalType = type; - if (type instanceof IntersectionType intersectionType) { - type = getMutableType(intersectionType); - avroRecord = ValueCreator.createRecordValue((RecordType) type); - } else if (type instanceof RecordType recordType) { - avroRecord = ValueCreator.createRecordValue(recordType); - } else { - throw new Exception("Type is not a valid record type"); - } - for (Schema.Field field : schema.getFields()) { - Object fieldData = rec.get(field.name()); - switch (field.schema().getType()) { - case MAP -> { - Type mapType = extractMapType(type); - avroRecord.put(StringUtils.fromString(field.name()), - visitMap((Map) rec.get(field.name()), mapType, field.schema())); - } - case ARRAY -> - avroRecord.put(StringUtils.fromString(field.name()), visitArray(field.schema(), - (GenericData.Array) rec.get(field.name()), type)); - case BYTES -> { - ByteBuffer byteBuffer = (ByteBuffer) rec.get(field.name()); - avroRecord.put(StringUtils.fromString(field.name()), - ValueCreator.createArrayValue(byteBuffer.array())); - } - case STRING -> - avroRecord.put(StringUtils.fromString(field.name()), - StringUtils.fromString(rec.get(field.name()).toString())); - case RECORD -> { - Type recType = extractRecordType((RecordType) type); - avroRecord.put(StringUtils.fromString(field.name()), - visitRecords(recType, field.schema(), (GenericRecord) rec.get(field.name()))); - } - case INT -> - avroRecord.put(StringUtils.fromString(field.name()), Long.parseLong(fieldData.toString())); - case FLOAT -> - avroRecord.put(StringUtils.fromString(field.name()), Double.parseDouble(fieldData.toString())); - case UNION -> - visitUnionRecords(type, avroRecord, field, fieldData); - default -> - avroRecord.put(StringUtils.fromString(field.name()), rec.get(field.name())); - } - } - if (originalType.isReadOnly()) { - avroRecord.freezeDirect(); - } - return avroRecord; - } - - @SuppressWarnings("unchecked") - private void visitUnionRecords(Type type, BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - for (Schema schemaType : field.schema().getTypes()) { - if (fieldData == null) { - avroRecord.put(StringUtils.fromString(field.name()), null); - break; - } - switch (schemaType.getType()) { - case BYTES -> { - if (fieldData instanceof ByteBuffer) { - BArray byteArray = ValueCreator.createArrayValue(((ByteBuffer) fieldData).array()); - avroRecord.put(StringUtils.fromString(field.name()), byteArray); - } - } - case FIXED -> { - if (fieldData instanceof GenericFixed) { - BArray byteArray = ValueCreator.createArrayValue(((GenericData.Fixed) fieldData).bytes()); - avroRecord.put(StringUtils.fromString(field.name()), byteArray); - } - } - case ARRAY -> { - if (fieldData instanceof GenericData.Array) { - Object[] objectArray = ((GenericData.Array) fieldData).toArray(); - if (schemaType.getElementType().getType().equals(Schema.Type.STRING) - || schemaType.getElementType().getType().equals(Schema.Type.ENUM)) { - BString[] stringArray = new BString[objectArray.length]; - BArray ballerinaArray = ValueCreator.createArrayValue(stringArray); - int i = 0; - for (Object obj : objectArray) { - stringArray[i] = StringUtils.fromString(obj.toString()); - i++; - } - avroRecord.put(StringUtils.fromString(field.name()), ballerinaArray); - } else { - avroRecord.put(StringUtils.fromString(field.name()), fieldData); - } - } - } - case MAP -> { - if (fieldData instanceof Map) { - BMap avroMap = ValueCreator.createMapValue(); - Object[] keys = ((Map) fieldData).keySet().toArray(); - for (Object key : keys) { - avroMap.put(StringUtils.fromString(key.toString()), - ((Map) fieldData).get(key)); - } - avroRecord.put(StringUtils.fromString(field.name()), avroMap); - } - } - case RECORD -> { - if (fieldData instanceof GenericRecord) { - avroRecord.put(StringUtils.fromString(field.name()), - visitRecords(type, schemaType, (GenericRecord) fieldData)); - } - } - case STRING -> { - if (fieldData instanceof Utf8) { - avroRecord.put(StringUtils.fromString(field.name()), - StringUtils.fromString(fieldData.toString())); - } - } - case INT, LONG -> { - if (fieldData instanceof Integer || fieldData instanceof Long) { - avroRecord.put(StringUtils.fromString(field.name()), - ((Number) fieldData).longValue()); - } - } - case FLOAT, DOUBLE -> { - if (fieldData instanceof Double) { - avroRecord.put(StringUtils.fromString(field.name()), fieldData); - } else { - avroRecord.put(StringUtils.fromString(field.name()), Double.parseDouble(fieldData.toString())); - } - } - case ENUM -> { - if (fieldData instanceof GenericEnumSymbol) { - avroRecord.put(StringUtils.fromString(field.name()), - StringUtils.fromString(fieldData.toString())); - } - } - default -> { - if (fieldData instanceof Boolean) { - avroRecord.put(StringUtils.fromString(field.name()), fieldData); - } - } - } - } - } - - private static Type extractMapType(Type type) { - Type mapType = type; - for (Map.Entry entry : ((RecordType) type).getFields().entrySet()) { - Field fieldValue = entry.getValue(); - if (fieldValue != null) { - Type fieldType = fieldValue.getFieldType(); - if (fieldType instanceof MapType) { - mapType = fieldType; - } else if (TypeUtils.getReferredType(fieldType) instanceof MapType) { - mapType = TypeUtils.getReferredType(fieldType); - } else if (fieldType instanceof IntersectionType) { - Type referredType = getMutableType((IntersectionType) fieldType); - if (referredType instanceof MapType) { - mapType = referredType; - } - } - } - } - return mapType; - } - - private static RecordType extractRecordType(RecordType type) { - Map fieldsMap = type.getFields(); - RecordType recType = type; - for (Map.Entry entry : fieldsMap.entrySet()) { - Field fieldValue = entry.getValue(); - if (fieldValue != null) { - Type fieldType = fieldValue.getFieldType(); - if (fieldType instanceof RecordType) { - recType = (RecordType) fieldType; - } else if (fieldType instanceof IntersectionType) { - Type getType = getMutableType((IntersectionType) fieldType); - if (getType instanceof RecordType) { - recType = (RecordType) getType; - } - } else if (TypeUtils.getReferredType(fieldType) instanceof RecordType) { - recType = (RecordType) TypeUtils.getReferredType(fieldType); - } - } - } - return recType; - } -} diff --git a/native/src/main/java/io/ballerina/lib/avro/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/visitor/SerializeVisitor.java deleted file mode 100644 index 8bfa327..0000000 --- a/native/src/main/java/io/ballerina/lib/avro/visitor/SerializeVisitor.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.ballerina.lib.avro.visitor; - -import io.ballerina.runtime.api.types.Type; -import io.ballerina.runtime.api.utils.StringUtils; -import io.ballerina.runtime.api.utils.TypeUtils; -import io.ballerina.runtime.api.values.BArray; -import io.ballerina.runtime.api.values.BMap; -import io.ballerina.runtime.api.values.BString; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericFixed; -import org.apache.avro.generic.GenericRecord; - -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static io.ballerina.lib.avro.Utils.ARRAY_TYPE; -import static io.ballerina.lib.avro.Utils.FLOAT_TYPE; -import static io.ballerina.lib.avro.Utils.INTEGER_TYPE; -import static io.ballerina.lib.avro.Utils.MAP_TYPE; -import static io.ballerina.lib.avro.Utils.RECORD_TYPE; -import static io.ballerina.lib.avro.Utils.STRING_TYPE; - -public class SerializeVisitor implements ISerializeVisitor { - - @Override - public String visitString(Object data) { - return ((BString) data).getValue(); - } - - @Override - public GenericRecord visitRecord(BMap data, Schema schema) throws Exception { - GenericRecord genericRecord = new GenericData.Record(schema); - for (Schema.Field field: schema.getFields()) { - Object fieldData = data.get(StringUtils.fromString(field.name())); - Schema.Type type = field.schema().getType(); - switch (type) { - case UNION -> - genericRecord.put(field.name(), visitUnion(fieldData, field)); - case RECORD -> - genericRecord.put(field.name(), visitRecord((BMap) fieldData, field.schema())); - case MAP -> - genericRecord.put(field.name(), visitMap((BMap) fieldData, field.schema())); - case ARRAY -> - genericRecord.put(field.name(), visitArray((BArray) fieldData, field.schema())); - case INT, LONG, FLOAT, DOUBLE, BOOLEAN, STRING, BYTES -> - genericRecord.put(field.name(), visitRecordPrimitives(fieldData, type)); - case ENUM -> - genericRecord.put(field.name(), visitEnum(fieldData, field.schema())); - default -> - genericRecord.put(field.name(), fieldData); - } - } - return genericRecord; - } - - private Object visitRecordPrimitives(Object data, Schema.Type type) { - switch (type) { - case INT -> { - return ((Long) data).intValue(); - } - case BYTES -> { - ByteBuffer byteBuffer = ByteBuffer.allocate(((BArray) data).getByteArray().length); - byteBuffer.put(((BArray) data).getByteArray()); - byteBuffer.position(0); - return byteBuffer; - } - case STRING -> { - return data.toString(); - } - default -> { - return data; - } - } - } - - @Override - public Map visitMap(BMap data, Schema schema) throws Exception { - Map avroMap = new HashMap<>(); - if (schema.getType().equals(Schema.Type.UNION)) { - for (Schema fieldSchema: schema.getTypes()) { - if (fieldSchema.getType().equals(Schema.Type.MAP)) { - schema = fieldSchema; - } - } - } - Schema.Type type = schema.getValueType().getType(); - for (Object value : data.getKeys()) { - switch (type) { - case INT, LONG, FLOAT, DOUBLE, BOOLEAN, STRING, BYTES -> - avroMap.put(value.toString(), visitRecordPrimitives(data.get(value), type)); - case RECORD -> - avroMap.put(value.toString(), visitRecord((BMap) data.get(value), schema.getValueType())); - case MAP -> - avroMap.put(value.toString(), visitMap((BMap) data.get(value), schema.getValueType())); - case ARRAY -> - avroMap.put(value.toString(), visitArray((BArray) data.get(value), schema.getValueType())); - case ENUM -> - avroMap.put(value.toString(), visitEnum(data.get(value), schema.getValueType())); - case FIXED -> - avroMap.put(value.toString(), visitFixed(data.get(value), schema.getValueType())); - default -> - throw new IllegalArgumentException("Unsupported schema type: " + type); - } - } - return avroMap; - } - - @Override - public Object visitBytes(Object data, Schema schema) { - if (schema.getType().equals(Schema.Type.UNION)) { - for (Schema fieldSchema: schema.getTypes()) { - if (fieldSchema.getType().equals(Schema.Type.BYTES)) { - return ByteBuffer.wrap(((BArray) data).getByteArray()); - } - } - } - return ByteBuffer.wrap(((BArray) data).getByteArray()); - } - @Override - public Object visitEnum(Object data, Schema schema) { - return new GenericData.EnumSymbol(schema, data); - } - - @Override - public GenericData.Fixed visitFixed(Object data, Schema schema) { - return new GenericData.Fixed(schema, ((BArray) data).getByteArray()); - } - - @Override - public GenericData.Array visitArray(BArray data, Schema schema) throws Exception { - GenericData.Array array = new GenericData.Array<>(data.size(), schema); - switch (schema.getElementType().getType()) { - case ARRAY -> { - Arrays.stream(data.getValues()) - .filter(Objects::nonNull) - .forEach(value -> { - try { - array.add(visitArray((BArray) value, schema.getElementType())); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return array; - } - case ENUM -> { - Arrays.stream((data.getValues() == null) ? data.getStringArray() : data.getValues()) - .filter(Objects::nonNull) - .forEach(value -> { - try { - array.add(new GenericData.EnumSymbol(schema.getElementType(), value)); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return array; - } - case STRING -> { - return visitStringArray(data, array); - } - case INT, LONG -> { - return visitLongArray(data, array); - } - case FLOAT, DOUBLE -> { - return visitDoubleArray(data, array); - } - case BOOLEAN -> { - return visitBooleanArray(data, array); - } - case MAP -> { - return visitMapArray(data, schema, array); - } - case RECORD -> { - return visitRecordArray(data, schema, array); - } - case FIXED -> { - return visitFixed(data, array, schema.getElementType()); - } - default -> { - return visitBytes(data, array); - } - } - } - - private static GenericData.Array visitRecordArray(BArray data, Schema schema, - GenericData.Array array) { - Arrays.stream(data.getValues()) - .filter(Objects::nonNull) - .forEach(record -> { - try { - array.add(new SerializeVisitor().visitRecord((BMap) record, schema.getElementType())); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return array; - } - - private static GenericData.Array visitMapArray(BArray data, Schema schema, - GenericData.Array array) { - Arrays.stream(data.getValues()) - .filter(Objects::nonNull) - .forEach(record -> { - try { - array.add(new SerializeVisitor().visitMap((BMap) record, schema.getElementType())); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return array; - } - - private GenericData.Array visitBytes(BArray data, GenericData.Array array) { - Arrays.stream(data.getValues()) - .filter(Objects::nonNull) - .forEach(bytes -> { - ByteBuffer byteBuffer = ByteBuffer.allocate(((BArray) bytes).getByteArray().length); - byteBuffer.put(((BArray) bytes).getByteArray()); - byteBuffer.position(0); - array.add(byteBuffer); - }); - return array; - } - - private GenericData.Array visitFixed(BArray data, GenericData.Array array, Schema schema) { - Arrays.stream(data.getValues()) - .filter(Objects::nonNull) - .forEach(bytes -> { - GenericFixed genericFixed = new GenericData.Fixed(schema, ((BArray) bytes).getByteArray()); - array.add(genericFixed); - }); - return array; - } - - private GenericData.Array visitBooleanArray(BArray data, GenericData.Array array) { - for (Object obj: data.getBooleanArray()) { - array.add(obj); - } - return array; - } - - private GenericData.Array visitDoubleArray(BArray data, GenericData.Array array) throws Exception { - try { - for (Object obj: data.getFloatArray()) { - array.add(obj); - } - return array; - } catch (Exception e) { - throw new Exception(e.getMessage()); - } - } - - private GenericData.Array visitLongArray(BArray data, GenericData.Array array) { - for (Object obj: data.getIntArray()) { - array.add(obj); - } - return array; - } - - private GenericData.Array visitStringArray(BArray data, GenericData.Array array) { - array.addAll(Arrays.asList(data.getStringArray())); - return array; - } - - private Object visitUnion(Object data, Schema.Field field) throws Exception { - Type typeName = TypeUtils.getType(data); - switch (typeName.getClass().getSimpleName()) { - case STRING_TYPE -> { - for (Schema schema : field.schema().getTypes()) { - if (schema.getType().equals(Schema.Type.ENUM)) { - return new GenericData.EnumSymbol(schema, data); - } - } - return data.toString(); - } - case ARRAY_TYPE -> { - for (Schema schema: field.schema().getTypes()) { - if (schema.getType().equals(Schema.Type.BYTES)) { - return ByteBuffer.wrap(((BArray) data).getByteArray()); - } else if (schema.getType().equals(Schema.Type.FIXED)) { - return new GenericData.Fixed(schema, ((BArray) data).getByteArray()); - } else if (schema.getType().equals(Schema.Type.ARRAY)) { - return visitArray((BArray) data, schema); - } - } - return visitArray((BArray) data, field.schema()); - } - case MAP_TYPE -> { - return visitMap((BMap) data, field.schema()); - } - case RECORD_TYPE -> { - return visitRecord((BMap) data, getRecordSchema(Schema.Type.RECORD, field.schema().getTypes())); - } - case INTEGER_TYPE -> { - for (Schema schema : field.schema().getTypes()) { - if (schema.getType().equals(Schema.Type.INT)) { - return ((Long) data).intValue(); - } - } - return data; - } - case FLOAT_TYPE -> { - for (Schema schema: field.schema().getTypes()) { - if (schema.getType().equals(Schema.Type.FLOAT)) { - return ((Double) data).floatValue(); - } - } - return data; - } - default -> { - return data; - } - } - } - - public static Schema getRecordSchema(Schema.Type givenType, List schemas) { - for (Schema schema: schemas) { - if (schema.getType().equals(Schema.Type.UNION)) { - getRecordSchema(givenType, schema.getTypes()); - } else if (schema.getType().equals(givenType)) { - return schema; - } - } - return null; - } -} From dab95d3de3208ccdc7279274421279a3b9a88b4d Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 09:09:38 +0530 Subject: [PATCH 19/47] Update generating serializer classes --- .../java/io/ballerina/lib/avro/serialize/MessageFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java index c9daf44..e1814fa 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java @@ -32,7 +32,7 @@ public static Serializer createMessage(Schema schema) { case MAP -> new MapSerializer(schema); case RECORD -> new RecordSerializer(schema); case BYTES -> new ByteSerializer(); - default -> new GenericSerializer(); + default -> new GenericSerializer(schema); }; } } From 3120fa0482a2d751d4f9491707f3efc2e4e0b95a Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 09:10:09 +0530 Subject: [PATCH 20/47] Add utils functions for union records into separate class --- .../deserialize/visitor/UnionRecordUtils.java | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java new file mode 100644 index 0000000..664cea7 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java @@ -0,0 +1,151 @@ +package io.ballerina.lib.avro.deserialize.visitor; + +import io.ballerina.lib.avro.deserialize.RecordDeserializer; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericEnumSymbol; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.util.Utf8; + +import java.nio.ByteBuffer; +import java.util.Map; + +public class UnionRecordUtils { + + public static void visitUnionRecords(Type type, BMap ballerinaRecord, + Schema.Field field, Object fieldData) throws Exception { + for (Schema schemaType : field.schema().getTypes()) { + if (fieldData == null) { + ballerinaRecord.put(StringUtils.fromString(field.name()), null); + break; + } + switch (schemaType.getType()) { + case BYTES: + handleBytesField(field, fieldData, ballerinaRecord); + break; + case FIXED: + handleFixedField(field, fieldData, ballerinaRecord); + break; + case ARRAY: + handleArrayField(field, fieldData, ballerinaRecord, schemaType); + break; + case MAP: + handleMapField(field, fieldData, ballerinaRecord); + break; + case RECORD: + handleRecordField(type, field, fieldData, ballerinaRecord, schemaType); + break; + case STRING: + handleStringField(field, fieldData, ballerinaRecord); + break; + case INT, LONG: + handleIntegerField(field, fieldData, ballerinaRecord); + break; + case FLOAT, DOUBLE: + handleFloatField(field, fieldData, ballerinaRecord); + break; + case ENUM: + handleEnumField(field, fieldData, ballerinaRecord); + break; + default: + handleDefaultField(field, fieldData, ballerinaRecord); + } + } + } + + private static void handleDefaultField(Schema.Field field, Object fieldData, + BMap ballerinaRecord) { + if (fieldData instanceof Boolean) { + ballerinaRecord.put(StringUtils.fromString(field.name()), fieldData); + } + } + + private static void handleEnumField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof GenericEnumSymbol) { + ballerinaRecord.put(StringUtils.fromString(field.name()), StringUtils.fromString(fieldData.toString())); + } + } + + private static void handleFloatField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof Double) { + ballerinaRecord.put(StringUtils.fromString(field.name()), fieldData); + } else { + ballerinaRecord.put(StringUtils.fromString(field.name()), Double.parseDouble(fieldData.toString())); + } + } + + private static void handleIntegerField(Schema.Field field, Object fieldData, + BMap ballerinaRecord) { + if (fieldData instanceof Integer || fieldData instanceof Long) { + ballerinaRecord.put(StringUtils.fromString(field.name()), ((Number) fieldData).longValue()); + } + } + + private static void handleStringField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof Utf8) { + ballerinaRecord.put(StringUtils.fromString(field.name()), StringUtils.fromString(fieldData.toString())); + } + } + + public static void handleRecordField(Type type, Schema.Field field, Object fieldData, + BMap ballerinaRecord, Schema schemaType) throws Exception { + if (fieldData instanceof GenericRecord) { + RecordDeserializer recordDes = new RecordDeserializer(schemaType, type); + Object fieldValue = recordDes.visit(new DeserializeVisitor(), (GenericRecord) fieldData); + ballerinaRecord.put(StringUtils.fromString(field.name()), fieldValue); + } + } + + private static void handleMapField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof Map) { + BMap avroMap = ValueCreator.createMapValue(); + Object[] keys = ((Map) fieldData).keySet().toArray(); + for (Object key : keys) { + avroMap.put(StringUtils.fromString(key.toString()), + ((Map) fieldData).get(key)); + } + ballerinaRecord.put(StringUtils.fromString(field.name()), avroMap); + } + } + + private static void handleBytesField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof ByteBuffer) { + BArray byteArray = ValueCreator.createArrayValue(((ByteBuffer) fieldData).array()); + ballerinaRecord.put(StringUtils.fromString(field.name()), byteArray); + } + } + + private static void handleFixedField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof GenericFixed) { + BArray byteArray = ValueCreator.createArrayValue(((GenericData.Fixed) fieldData).bytes()); + ballerinaRecord.put(StringUtils.fromString(field.name()), byteArray); + } + } + + private static void handleArrayField(Schema.Field field, Object fieldData, + BMap ballerinaRecord, Schema schemaType) { + if (fieldData instanceof GenericData.Array) { + Object[] objectArray = ((GenericData.Array) fieldData).toArray(); + if (schemaType.getElementType().getType().equals(Schema.Type.STRING) + || schemaType.getElementType().getType().equals(Schema.Type.ENUM)) { + BString[] stringArray = new BString[objectArray.length]; + BArray ballerinaArray = ValueCreator.createArrayValue(stringArray); + int i = 0; + for (Object obj : objectArray) { + stringArray[i] = StringUtils.fromString(obj.toString()); + i++; + } + ballerinaRecord.put(StringUtils.fromString(field.name()), ballerinaArray); + } else { + ballerinaRecord.put(StringUtils.fromString(field.name()), fieldData); + } + } + } +} From 0e9429edba1d84eb55ac6b16e7041bc9345e6633 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 09:10:41 +0530 Subject: [PATCH 21/47] Update visitors to generate deserialized values --- .../visitor/DeserializeArrayVisitor.java | 30 +++++ .../visitor/DeserializeRecordVisitor.java | 108 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java new file mode 100644 index 0000000..26ea4cf --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java @@ -0,0 +1,30 @@ +package io.ballerina.lib.avro.deserialize.visitor; + +import io.ballerina.lib.avro.deserialize.ArrayDeserializer; +import io.ballerina.lib.avro.deserialize.Deserializer; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.ArrayType; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.generic.GenericData; + +public class DeserializeArrayVisitor extends DeserializeVisitor { + + public Object visit(ArrayDeserializer arrayDeserializer, GenericData.Array data) throws Exception { + Object[] objects = new Object[data.size()]; + Type arrayType = ((ArrayType) arrayDeserializer.getType()).getElementType(); + int index = 0; + for (Object element : data) { + GenericData.Array dataArray = (GenericData.Array) element; + Type arrType = arrayType instanceof ArrayType ? arrayType : arrayDeserializer.getType(); + objects[index++] = visitNestedArray(new ArrayDeserializer(arrayDeserializer.getSchema().getElementType(), + arrType), dataArray); + } + return ValueCreator.createArrayValue(objects, (ArrayType) arrayDeserializer.getType()); + } + + public Object visitNestedArray(ArrayDeserializer arrayDeserializer, + GenericData.Array data) throws Exception { + Deserializer deserializer = createDeserializer(arrayDeserializer.getSchema(), arrayDeserializer.getType()); + return deserializer.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java new file mode 100644 index 0000000..365f254 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java @@ -0,0 +1,108 @@ +package io.ballerina.lib.avro.deserialize.visitor; + +import io.ballerina.lib.avro.deserialize.ArrayDeserializer; +import io.ballerina.lib.avro.deserialize.MapDeserializer; +import io.ballerina.lib.avro.deserialize.RecordDeserializer; +import io.ballerina.lib.avro.deserialize.StringDeserializer; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.IntersectionType; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +import java.nio.ByteBuffer; +import java.util.Map; + +import static io.ballerina.lib.avro.Utils.getMutableType; +import static io.ballerina.lib.avro.deserialize.visitor.UnionRecordUtils.visitUnionRecords; + +public class DeserializeRecordVisitor extends DeserializeVisitor { + + public BMap visit(RecordDeserializer recordDeserializer, GenericRecord rec) throws Exception { + Type originalType = recordDeserializer.getType(); + Type type = recordDeserializer.getType(); + Schema schema = recordDeserializer.getSchema(); + BMap avroRecord = createAvroRecord(type); + for (Schema.Field field : schema.getFields()) { + Object fieldData = rec.get(field.name()); + switch (field.schema().getType()) { + case MAP -> + processMapField(avroRecord, field, fieldData); + case ARRAY -> + processArrayField(avroRecord, field, fieldData); + case BYTES -> + processBytesField(avroRecord, field, fieldData); + case RECORD -> + processRecordField(avroRecord, field, fieldData); + case STRING -> + processStringField(avroRecord, field, fieldData); + case INT -> + avroRecord.put(StringUtils.fromString(field.name()), Long.parseLong(fieldData.toString())); + case FLOAT -> + avroRecord.put(StringUtils.fromString(field.name()), Double.parseDouble(fieldData.toString())); + case UNION -> + processUnionField(type, avroRecord, field, fieldData); + default -> + avroRecord.put(StringUtils.fromString(field.name()), fieldData); + } + } + + if (originalType.isReadOnly()) { + avroRecord.freezeDirect(); + } + return avroRecord; + } + + private BMap createAvroRecord(Type type) { + if (type instanceof IntersectionType) { + type = getMutableType((IntersectionType) type); + } + return ValueCreator.createRecordValue((RecordType) type); + } + + private void processMapField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + Type mapType = extractMapType(avroRecord.getType()); + MapDeserializer mapDeserializer = new MapDeserializer(field.schema(), mapType); + Object fieldValue = mapDeserializer.visit(this, (Map) fieldData); + avroRecord.put(StringUtils.fromString(field.name()), fieldValue); + } + + private void processArrayField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + ArrayDeserializer arrayDes = new ArrayDeserializer(field.schema(), avroRecord.getType()); + Object fieldValue = arrayDes.visit(this, (GenericData.Array) fieldData); + avroRecord.put(StringUtils.fromString(field.name()), fieldValue); + } + + private void processBytesField(BMap avroRecord, Schema.Field field, Object fieldData) { + ByteBuffer byteBuffer = (ByteBuffer) fieldData; + Object fieldValue = ValueCreator.createArrayValue(byteBuffer.array()); + avroRecord.put(StringUtils.fromString(field.name()), fieldValue); + } + + private void processRecordField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + Type recType = extractRecordType((RecordType) avroRecord.getType()); + RecordDeserializer recordDes = new RecordDeserializer(field.schema(), recType); + Object fieldValue = recordDes.visit(this, (GenericRecord) fieldData); + avroRecord.put(StringUtils.fromString(field.name()), fieldValue); + } + + private void processStringField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + StringDeserializer stringDes = new StringDeserializer(); + Object fieldValue = stringDes.visit(this, fieldData); + avroRecord.put(StringUtils.fromString(field.name()), fieldValue); + } + + private void processUnionField(Type type, BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + visitUnionRecords(type, avroRecord, field, fieldData); + } +} From 7b4b413b5e86a2430416c36992db960979ed3f6e Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 09:11:02 +0530 Subject: [PATCH 22/47] Apply review comments --- .../src/main/java/io/ballerina/lib/avro/Avro.java | 11 +++++------ .../main/java/io/ballerina/lib/avro/Utils.java | 15 +++++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/Avro.java b/native/src/main/java/io/ballerina/lib/avro/Avro.java index ea65f32..efd3561 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Avro.java +++ b/native/src/main/java/io/ballerina/lib/avro/Avro.java @@ -20,10 +20,10 @@ import io.ballerina.lib.avro.deserialize.DeserializeFactory; import io.ballerina.lib.avro.deserialize.Deserializer; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; import io.ballerina.lib.avro.serialize.MessageFactory; import io.ballerina.lib.avro.serialize.Serializer; -import io.ballerina.lib.avro.visitor.DeserializeVisitor; -import io.ballerina.lib.avro.visitor.SerializeVisitor; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import io.ballerina.runtime.api.creators.ValueCreator; import io.ballerina.runtime.api.values.BArray; import io.ballerina.runtime.api.values.BObject; @@ -62,13 +62,12 @@ public static Object toAvro(BObject schemaObject, Object data) { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { SerializeVisitor serializeVisitor = new SerializeVisitor(); Serializer serializer = MessageFactory.createMessage(schema); - Object avroData = Objects.requireNonNull(serializer).generateMessage(serializeVisitor, data); + Object avroData = Objects.requireNonNull(serializer).convert(serializeVisitor, data); DatumWriter writer = new GenericDatumWriter<>(schema); BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null); writer.write(avroData, encoder); encoder.flush(); - byte[] bytes = outputStream.toByteArray(); - return ValueCreator.createArrayValue(bytes); + return ValueCreator.createArrayValue(outputStream.toByteArray()); } catch (Exception e) { return Utils.createError(SERIALIZATION_ERROR, e); } @@ -82,7 +81,7 @@ public static Object fromAvro(BObject schemaObject, BArray payload, BTypedesc ty Object data = datumReader.read(payload, decoder); DeserializeVisitor deserializeVisitor = new DeserializeVisitor(); Deserializer deserializer = DeserializeFactory.generateDeserializer(schema, typeParam.getDescribingType()); - return Objects.requireNonNull(deserializer).fromAvroMessage(deserializeVisitor, data, schema); + return Objects.requireNonNull(deserializer).fromAvro(deserializeVisitor, data); } catch (Exception e) { return createError(DESERIALIZATION_ERROR, e); } diff --git a/native/src/main/java/io/ballerina/lib/avro/Utils.java b/native/src/main/java/io/ballerina/lib/avro/Utils.java index 00cfa3a..75c730c 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Utils.java +++ b/native/src/main/java/io/ballerina/lib/avro/Utils.java @@ -38,12 +38,15 @@ private Utils() { public static final String ERROR_TYPE = "Error"; public static final String SERIALIZATION_ERROR = "Avro serialization error"; public static final String DESERIALIZATION_ERROR = "Avro deserialization error"; - public static final String STRING_TYPE = "BStringType"; - public static final String ARRAY_TYPE = "BArrayType"; - public static final String MAP_TYPE = "BMapType"; - public static final String RECORD_TYPE = "BRecordType"; - public static final String INTEGER_TYPE = "BIntegerType"; - public static final String FLOAT_TYPE = "BFloatType"; + public static final int STRING_TYPE = 5; + public static final int ARRAY_TYPE = 32; + public static final int MAP_TYPE = 27; + public static final int INTERSECTION_TYPE = 34; + public static final int REFERENCE_TYPE = 53; + public static final int RECORD_TYPE = 24; + public static final int INTEGER_TYPE = 1; + public static final int FLOAT_TYPE = 3; + public static final int BOOLEAN_TYPE = 6; public static BError createError(String message, Throwable throwable) { BError cause = ErrorCreator.createError(throwable); From 6fe6a3c1a2bea8c590e85b0386487775e2fab816 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 09:11:25 +0530 Subject: [PATCH 23/47] Update tests for new use cases --- ballerina/tests/array_tests.bal | 25 +++ ballerina/tests/byte_tests.bal | 2 +- ballerina/tests/map_tests.bal | 276 ++++++++++++++++++++++++------- ballerina/tests/record_tests.bal | 61 ++++++- ballerina/tests/types.bal | 9 +- 5 files changed, 312 insertions(+), 61 deletions(-) diff --git a/ballerina/tests/array_tests.bal b/ballerina/tests/array_tests.bal index bb90870..3cd3a08 100644 --- a/ballerina/tests/array_tests.bal +++ b/ballerina/tests/array_tests.bal @@ -59,6 +59,31 @@ public isolated function testStringArrays() returns error? { @test:Config { groups: ["array", "string"] } +public isolated function testArrayOfStringArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "stringArray", + "namespace": "data", + "items": { + "type": "array", + "name" : "strings", + "namespace": "data", + "items": "string" + } + }`; + + string[][] colors = [["red", "green", "blue"]]; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + string[][] deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, colors); +} + +@test:Config { + groups: ["array", "string", "enn"] +} public isolated function testEnumArrays() returns error? { string schema = string ` { diff --git a/ballerina/tests/byte_tests.bal b/ballerina/tests/byte_tests.bal index 4d434da..724ff0d 100644 --- a/ballerina/tests/byte_tests.bal +++ b/ballerina/tests/byte_tests.bal @@ -82,7 +82,7 @@ public isolated function testBytes() returns error? { } @test:Config { - groups: ["record", "maps", "bytes"] + groups: ["record", "map", "bytes"] } public isolated function testNestedRecordsWithBytes() returns error? { string schema = string ` diff --git a/ballerina/tests/map_tests.bal b/ballerina/tests/map_tests.bal index 26738ab..d00211c 100644 --- a/ballerina/tests/map_tests.bal +++ b/ballerina/tests/map_tests.bal @@ -17,7 +17,7 @@ import ballerina/test; @test:Config { - groups: ["maps", "bytes"] + groups: ["map", "bytes"] } public isolated function testMapsWithBytes() returns error? { string schema = string ` @@ -35,7 +35,7 @@ public isolated function testMapsWithBytes() returns error? { } @test:Config { - groups: ["maps", "fixed"] + groups: ["map", "fixed"] } public isolated function testMapsWithFixed() returns error? { string schema = string ` @@ -57,7 +57,7 @@ public isolated function testMapsWithFixed() returns error? { } @test:Config { - groups: ["maps", "fixed"] + groups: ["map", "fixed"] } public isolated function testMapsOfFixedMaps() returns error? { string schema = string ` @@ -87,7 +87,7 @@ public isolated function testMapsOfFixedMaps() returns error? { } @test:Config { - groups: ["maps", "record"] + groups: ["map", "record"] } public isolated function testMapsWithRecords() returns error? { string schema = string ` @@ -138,7 +138,7 @@ public isolated function testMapsWithRecords() returns error? { } @test:Config { - groups: ["maps"] + groups: ["map"] } public isolated function testMapsWithInt() returns error? { string schema = string ` @@ -157,7 +157,7 @@ public isolated function testMapsWithInt() returns error? { } @test:Config { - groups: ["maps", "enum"] + groups: ["map", "enum", "lk"] } public isolated function testMapsWithEnum() returns error? { string schema = string ` @@ -180,7 +180,7 @@ public isolated function testMapsWithEnum() returns error? { } @test:Config { - groups: ["maps", "enum", "array"] + groups: ["map", "enum", "array"] } public isolated function testMapsWithEnumArrays() returns error? { string schema = string ` @@ -208,7 +208,7 @@ public isolated function testMapsWithEnumArrays() returns error? { } @test:Config { - groups: ["maps", "float"] + groups: ["map", "float"] } public isolated function testMapsWithFloat() returns error? { string schema = string ` @@ -227,7 +227,7 @@ public isolated function testMapsWithFloat() returns error? { } @test:Config { - groups: ["maps", "double"] + groups: ["map", "double"] } public isolated function testMapsWithDouble() returns error? { string schema = string ` @@ -246,7 +246,7 @@ public isolated function testMapsWithDouble() returns error? { } @test:Config { - groups: ["maps", "double", "array"] + groups: ["map", "double", "array"] } public isolated function testMapsWithDoubleArray() returns error? { string schema = string ` @@ -270,7 +270,7 @@ public isolated function testMapsWithDoubleArray() returns error? { } @test:Config { - groups: ["maps", "long"] + groups: ["map", "long"] } public isolated function testMapsWithLong() returns error? { string schema = string ` @@ -289,7 +289,7 @@ public isolated function testMapsWithLong() returns error? { } @test:Config { - groups: ["maps", "string"] + groups: ["map", "string"] } public isolated function testMapsWithStrings() returns error? { string schema = string ` @@ -308,7 +308,25 @@ public isolated function testMapsWithStrings() returns error? { } @test:Config { - groups: ["maps", "boolean"] + groups: ["map", "errors"] +} +public isolated function testMapsWithUnionTypes() returns error? { + string schema = string ` + { + "type": "map", + "values": ["string", "int"], + "default": {} + }`; + + map colors = {"red": "2", "green": "435", "blue": "2034723"}; + + Schema avro = check new (schema); + byte[]|Error encode = avro.toAvro(colors); + test:assertTrue(encode is Error); +} + +@test:Config { + groups: ["map", "boolean", "ssq"] } public isolated function testMapsWithBoolean() returns error? { string schema = string ` @@ -327,7 +345,7 @@ public isolated function testMapsWithBoolean() returns error? { } @test:Config { - groups: ["maps"] + groups: ["map"] } public isolated function testMapsWithMaps() returns error? { string schema = string ` @@ -353,7 +371,7 @@ public isolated function testMapsWithMaps() returns error? { } @test:Config { - groups: ["maps", "k"] + groups: ["map", "k"] } public isolated function testMapsWithNestedMaps() returns error? { string schema = string ` @@ -382,7 +400,7 @@ public isolated function testMapsWithNestedMaps() returns error? { } @test:Config { - groups: ["maps", "long"] + groups: ["map", "long", "qwe"] } public isolated function testMapsWithLongArray() returns error? { string schema = string ` @@ -404,7 +422,7 @@ public isolated function testMapsWithLongArray() returns error? { } @test:Config { - groups: ["maps", "int", "az"] + groups: ["map", "int", "az"] } public isolated function testMapsWithIntArray() returns error? { string schema = string ` @@ -426,7 +444,7 @@ public isolated function testMapsWithIntArray() returns error? { } @test:Config { - groups: ["maps", "float"] + groups: ["map", "float"] } public isolated function testMapsWithFloatArray() returns error? { string schema = string ` @@ -439,7 +457,6 @@ public isolated function testMapsWithFloatArray() returns error? { "default": {} }`; - // 207.234345 map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; Schema avro = check new (schema); @@ -449,7 +466,7 @@ public isolated function testMapsWithFloatArray() returns error? { } @test:Config { - groups: ["maps", "string"] + groups: ["map", "string"] } public isolated function testMapsWithStringArray() returns error? { string schema = string ` @@ -471,7 +488,117 @@ public isolated function testMapsWithStringArray() returns error? { } @test:Config { - groups: ["maps", "bytes"] + groups: ["map", "string", "null", "union", "sbs"] +} +public isolated function testMapsWithUnionArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": ["string", "null"] + }, + "default": {} + }`; + + map colors = {"red": ["252", "122", "41"], "green": ["235", "163", "23"], "blue": ["207", "123"]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["map", "int", "null", "union"] +} +public isolated function testMapsWithUnionIntArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": ["int", "null"] + }, + "default": {} + }`; + + map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["map", "long", "null", "union", "sxs"] +} +public isolated function testMapsWithUnionLongArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": ["long", "null"] + }, + "default": {} + }`; + + map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["map", "float", "null", "union", "sss"] +} +public isolated function testMapsWithUnionFloatArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": ["float", "null"] + }, + "default": {} + }`; + + map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["map", "float", "null", "union"] +} +public isolated function testMapsWithUnionDoubleArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": ["double", "null"] + }, + "default": {} + }`; + + map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encode); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["map", "bytes"] } public isolated function testMapsWithBytesArray() returns error? { string schema = string ` @@ -493,7 +620,7 @@ public isolated function testMapsWithBytesArray() returns error? { } @test:Config { - groups: ["maps", "bytes"] + groups: ["map", "bytes"] } public isolated function testMapsWithFixedArray() returns error? { string schema = string ` @@ -519,7 +646,7 @@ public isolated function testMapsWithFixedArray() returns error? { } @test:Config { - groups: ["maps", "boolean"] + groups: ["map", "boolean"] } public isolated function testMapsWithBooleanArray() returns error? { string schema = string ` @@ -541,7 +668,7 @@ public isolated function testMapsWithBooleanArray() returns error? { } @test:Config { - groups: ["maps", "record"] + groups: ["map", "record", "gg"] } public isolated function testMapsWithRecordArray() returns error? { string schema = string ` @@ -550,52 +677,85 @@ public isolated function testMapsWithRecordArray() returns error? { "values": { "type": "array", "items": { - "type": "record", - "name": "Instructor", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "student", - "type": ["null", { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "subject", - "type": "string" - } - ] - }] - } - ] + "type": "array", + "items": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + } } }, "default": {} }`; - - - map instructors = { - "john": [{name: "John", student: {name: "Alice", subject: "Math"}}, {name: "John", student: {name: "Alice", subject: "Math"}}], - "doe": [{name: "Doe", student: {name: "Bob", subject: "Science"}}, {name: "Doe", student: {name: "Bob", subject: "Science"}}], - "jane": [{name: "Jane", student: {name: "Charlie", subject: "English"}}, {name: "Jane", student: {name: "Charlie", subject: "English"}}] + map instructors = { + "john": [[{name: "John", student: {name: "Alice", subject: "Math"}}, {name: "John", student: {name: "Alice", subject: "Math"}}]], + "doe": [[{name: "Doe", student: {name: "Bob", subject: "Science"}}, {name: "Doe", student: {name: "Bob", subject: "Science"}}]], + "jane": [[{name: "Jane", student: {name: "Charlie", subject: "English"}}, {name: "Jane", student: {name: "Charlie", subject: "English"}}]] }; Schema avro = check new (schema); byte[] encode = check avro.toAvro(instructors); - map deserialize = check avro.fromAvro(encode); + map deserialize = check avro.fromAvro(encode); test:assertEquals(instructors, deserialize); } @test:Config { - groups: ["maps", "record"] + groups: ["map", "record"] +} +public isolated function testArrayOfStringArrayMaps() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "name" : "stringArray", + "namespace": "data", + "items": { + "type": "array", + "name" : "strings", + "namespace": "data", + "items": "string" + } + } + }`; + + map colors = { + "red": [["red", "green", "blue"], ["red", "green", "blue"], ["red", "green", "blue"]], + "green": [["red", "green", "blue"], ["red", "green", "blue"], ["red", "green", "blue"]], + "blue": [["red", "green", "blue"], ["red", "green", "blue"], ["red", "green", "blue"]] + }; + + Schema avro = check new (schema); + byte[] encode = check avro.toAvro(colors); + map deserializeJson = check avro.fromAvro(encode); + test:assertEquals(deserializeJson, colors); +} + +@test:Config { + groups: ["map", "record"] } public isolated function testMapsWithNestedRecordMaps() returns error? { string schema = string ` @@ -672,7 +832,7 @@ public isolated function testMapsWithNestedRecordMaps() returns error? { @test:Config { - groups: ["maps", "record", "wx"] + groups: ["map", "record"] } public isolated function testMapsWithNestedRecordArrayMaps() returns error? { string schema = string ` diff --git a/ballerina/tests/record_tests.bal b/ballerina/tests/record_tests.bal index 17903c4..e986434 100644 --- a/ballerina/tests/record_tests.bal +++ b/ballerina/tests/record_tests.bal @@ -210,6 +210,65 @@ public isolated function testArraysInRecordsWithInvalidSchema() returns error? { test:assertTrue(deserialize is Error); } +@test:Config { + groups: ["record", "union", "pm"] +} +public isolated function testRecordsWithStringRecordUnionType() returns error? { + string schema = string ` + { + "type": "record", + "name": "Course", + "namespace": "example.avro", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "value", + "type": "float" + }, + { + "name": "credits", + "type": ["null", "int"] + }, + { + "name": "student", + "type": [ + "null", + { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + }, + "string" + ] + } + ] + }`; + + MultipleUnionRecord course = { + name: "data", + value: 0.0, + credits: 5, + student: string `{name: "Jon", subject: "geo"}adsk` + }; + + Schema avro = check new (schema); + byte[] serialize = check avro.toAvro(course); + MultipleUnionRecord deserialize = check avro.fromAvro(serialize); + test:assertEquals(deserialize, course); +} + @test:Config { groups: ["record", "union"] } @@ -494,7 +553,7 @@ public isolated function testOptionalMultipleFieldsInRecords() returns error? { "type": ["null", "boolean"] }, { - "name": "maps", + "name": "map", "type": [ "null", { diff --git a/ballerina/tests/types.bal b/ballerina/tests/types.bal index ef61410..9b3f2c4 100644 --- a/ballerina/tests/types.bal +++ b/ballerina/tests/types.bal @@ -46,7 +46,7 @@ public type Course record { public type Instructor record { string? name; - Student? student; + Student? & readonly student; }; public type Lecturer record { @@ -108,6 +108,13 @@ type UnionRecord record { StudentRecord? student; }; +type MultipleUnionRecord record { + string? name; + int? credits; + float value; + string|StudentRecord? student; +}; + type StudentRecord record { string? name; string? subject; From 527f35620aa4a281ae0fd81011dbc8259e8af9f4 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 10:09:23 +0530 Subject: [PATCH 24/47] Fix review comment to use type tags --- native/src/main/java/io/ballerina/lib/avro/Utils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/Utils.java b/native/src/main/java/io/ballerina/lib/avro/Utils.java index 75c730c..3a462af 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Utils.java +++ b/native/src/main/java/io/ballerina/lib/avro/Utils.java @@ -18,9 +18,9 @@ package io.ballerina.lib.avro; +import io.ballerina.runtime.api.TypeTags; import io.ballerina.runtime.api.creators.ErrorCreator; import io.ballerina.runtime.api.types.IntersectionType; -import io.ballerina.runtime.api.types.MapType; import io.ballerina.runtime.api.types.Type; import io.ballerina.runtime.api.types.UnionType; import io.ballerina.runtime.api.utils.StringUtils; @@ -56,9 +56,9 @@ public static BError createError(String message, Throwable throwable) { public static Type getMutableType(IntersectionType intersectionType) { for (Type type : intersectionType.getConstituentTypes()) { Type referredType = TypeUtils.getImpliedType(type); - if (referredType instanceof UnionType) { + if (referredType.getTag() == TypeTags.UNION_TAG) { for (Type elementType : ((UnionType) referredType).getMemberTypes()) { - if (elementType instanceof MapType) { + if (elementType.getTag() == TypeTags.MAP_TAG) { return elementType; } } From 5abe679550f4b3ea1487c72d33ade66ce8832e27 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 10:51:57 +0530 Subject: [PATCH 25/47] Update API in Deserializer classes --- .../main/java/io/ballerina/lib/avro/Avro.java | 2 +- .../avro/deserialize/ArrayDeserializer.java | 2 +- .../avro/deserialize/ByteDeserializer.java | 2 +- .../lib/avro/deserialize/Deserializer.java | 2 +- .../avro/deserialize/DoubleDeserializer.java | 4 +- .../avro/deserialize/EnumDeserializer.java | 3 +- .../avro/deserialize/GenericDeserializer.java | 4 +- .../lib/avro/deserialize/MapDeserializer.java | 7 +-- .../avro/deserialize/NullDeserializer.java | 3 +- .../avro/deserialize/RecordDeserializer.java | 7 +-- .../avro/deserialize/StringDeserializer.java | 7 +-- .../avro/deserialize/UnionDeserializer.java | 5 +- .../visitor/DeserializeVisitor.java | 59 ++++++++++++------- 13 files changed, 54 insertions(+), 53 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/Avro.java b/native/src/main/java/io/ballerina/lib/avro/Avro.java index efd3561..41e06fa 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Avro.java +++ b/native/src/main/java/io/ballerina/lib/avro/Avro.java @@ -81,7 +81,7 @@ public static Object fromAvro(BObject schemaObject, BArray payload, BTypedesc ty Object data = datumReader.read(payload, decoder); DeserializeVisitor deserializeVisitor = new DeserializeVisitor(); Deserializer deserializer = DeserializeFactory.generateDeserializer(schema, typeParam.getDescribingType()); - return Objects.requireNonNull(deserializer).fromAvro(deserializeVisitor, data); + return Objects.requireNonNull(deserializer).visit(deserializeVisitor, data); } catch (Exception e) { return createError(DESERIALIZATION_ERROR, e); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java index fcfb483..400554d 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java @@ -30,7 +30,7 @@ public ArrayDeserializer(Schema schema, Type type) { } @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception { + public Object visit(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visit(this, (GenericData.Array) data); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java index 508bd16..7f61a23 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java @@ -5,7 +5,7 @@ public class ByteDeserializer extends Deserializer { @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception { + public Object visit(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visitBytes(data); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java index d3e384b..efe498c 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java @@ -52,6 +52,6 @@ public Type getType() { return TypeUtils.getReferredType(type); } - public abstract Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception; + public abstract Object visit(DeserializeVisitor visitor, Object data) throws Exception; public abstract Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception; } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java index 9a9bac9..d9f9c86 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java @@ -24,12 +24,12 @@ public class DoubleDeserializer extends Deserializer { @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) { + public Object visit(DeserializeVisitor visitor, Object data) { return visitor.visitDouble(data); } @Override public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { - return null; + return visitor.visitDouble(data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java index 60664c3..c0c06c3 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java @@ -11,7 +11,7 @@ public EnumDeserializer(Type type) { } @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) { + public Object visit(DeserializeVisitor visitor, Object data) { return visitor.visit(this, (GenericData.Array) data); } @@ -19,5 +19,4 @@ public Object fromAvro(DeserializeVisitor visitor, Object data) { public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visit(this, data); } - } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java index f4ca25d..c85ea70 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java @@ -30,8 +30,8 @@ public GenericDeserializer(Schema schema, Type type) { } @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) { - return data; + public Object visit(DeserializeVisitor visitor, Object data) { + return visitor.visit(this, data); } @Override diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java index e259a6c..b505327 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java @@ -32,15 +32,10 @@ public MapDeserializer(Schema schema, Type type) { } @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception { + public Object visit(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visit(this, (Map) data); } - public Object visit(DeserializeVisitor visitor, Map data) throws Exception { - return visitor.visit(this, data); - } - - @Override public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return null; } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java index 8dbafd4..7218a12 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java @@ -24,14 +24,13 @@ public class NullDeserializer extends Deserializer { @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception { + public Object visit(DeserializeVisitor visitor, Object data) throws Exception { if (data != null) { throw new Exception("The value does not match with the null schema"); } return null; } - @Override public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return null; } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java index 0cd21f6..2f0d842 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java @@ -31,16 +31,11 @@ public RecordDeserializer(Schema schema, Type type) { } @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) throws Exception { + public Object visit(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visit(this, (GenericRecord) data); } - @Override public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visit(this, data); } - - public Object visit(DeserializeVisitor visitor, GenericRecord data) throws Exception { - return visitor.visit(this, data); - } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java index 873f2f3..303613c 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java @@ -24,16 +24,11 @@ public class StringDeserializer extends Deserializer { @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) { + public Object visit(DeserializeVisitor visitor, Object data) { return visitor.visitString(data); } - @Override public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return null; } - - public Object visit(DeserializeVisitor visitor, Object data) throws Exception { - return visitor.visitString(data); - } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java index 51901e0..51f8148 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java @@ -12,11 +12,10 @@ public UnionDeserializer(Schema schema, Type type) { } @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) { - return null; + public Object visit(DeserializeVisitor visitor, Object data) throws Exception { + return visitor.visit(this, (GenericData.Array) data); } - @Override public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visit(this, data); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java index e1d8cfd..4d3cd81 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -75,7 +75,7 @@ public static Deserializer createDeserializer(Schema schema, Type type) { case RECORD -> new RecordDeserializer(schema, type); case FIXED -> - new FixedDeserializer(type); + new FixedDeserializer(schema, type); default -> new GenericDeserializer(schema, type); }; @@ -149,27 +149,32 @@ public BMap visit(MapDeserializer mapDeserializer, Map) ValueUtils.convert(avroRecord, type); } - public Object visit(GenericDeserializer genericDeserializer, GenericData.Array data) { + public Object visit(GenericDeserializer genericDeserializer, Object data) { Schema schema = genericDeserializer.getSchema(); - switch (schema.getElementType().getType()) { - case STRING -> { - return visitStringArray(data); - } - case INT -> { - return visitIntArray(data); - } - case LONG -> { - return visitLongArray(data); - } - case FLOAT, DOUBLE -> { - return visitDoubleArray(data); - } - case BOOLEAN -> { - return visitBooleanArray(data); - } - default -> { - return visitBytesArray(data, genericDeserializer.getType()); + if (schema.getType().equals(Schema.Type.ARRAY)) { + GenericData.Array array = (GenericData.Array) data; + switch (schema.getElementType().getType()) { + case STRING -> { + return visitStringArray(array); + } + case INT -> { + return visitIntArray(array); + } + case LONG -> { + return visitLongArray(array); + } + case FLOAT, DOUBLE -> { + return visitDoubleArray(array); + } + case BOOLEAN -> { + return visitBooleanArray(array); + } + default -> { + return visitBytesArray(array, genericDeserializer.getType()); + } } + } else { + return data; } } @@ -321,6 +326,20 @@ public BArray visit(EnumDeserializer enumDeserializer, GenericData.Array return ValueCreator.createArrayValue(enums, (ArrayType) enumDeserializer.getType()); } + public Object visit(FixedDeserializer fixedDeserializer, Object data) { + if (fixedDeserializer.getSchema().getType().equals(Schema.Type.ARRAY)) { + GenericData.Array array = (GenericData.Array) data; + Type type = fixedDeserializer.getType(); + List values = new ArrayList<>(); + for (Object datum : array) { + values.add(visitFixed(datum)); + } + return ValueCreator.createArrayValue(values.toArray(new BArray[array.size()]), (ArrayType) type); + } else { + return visitFixed(data); + } + } + public BArray visit(FixedDeserializer fixedDeserializer, GenericData.Array data) { Type type = fixedDeserializer.getType(); List values = new ArrayList<>(); From 1d0602396655415c463482c125931512d2d951d4 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 10:52:12 +0530 Subject: [PATCH 26/47] Update API to deserialize fixed data --- .../lib/avro/deserialize/DeserializeFactory.java | 2 +- .../lib/avro/deserialize/FixedDeserializer.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java index b8c1749..ee2337b 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java @@ -29,7 +29,7 @@ public static Deserializer generateDeserializer(Schema schema, Type type) { case FLOAT, DOUBLE -> new DoubleDeserializer(); case STRING, ENUM -> new StringDeserializer(); case ARRAY -> new ArrayDeserializer(schema, type); - case FIXED -> new FixedDeserializer(type); + case FIXED -> new FixedDeserializer(schema, type); case MAP -> new MapDeserializer(schema, type); case RECORD -> new RecordDeserializer(schema, type); case BYTES -> new ByteDeserializer(); diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java index cd358eb..208ff42 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java @@ -20,22 +20,22 @@ import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; public class FixedDeserializer extends Deserializer { - public FixedDeserializer(Type type) { - super(type); + public FixedDeserializer(Schema schema, Type type) { + super(schema, type); } @Override - public Object fromAvro(DeserializeVisitor visitor, Object data) { - return visitor.visitFixed(data); + public Object visit(DeserializeVisitor visitor, Object data) { + return visitor.visit(this, data); } @Override public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visit(this, data); } - } From a5fef53404812dc86b6f3e1f1be201b32f0f5c56 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 11:29:41 +0530 Subject: [PATCH 27/47] Update ser/des classes for primitive types --- .../avro/deserialize/DeserializeFactory.java | 2 +- ...ializer.java => PrimitiveDeserializer.java} | 4 ++-- .../lib/avro/serialize/MessageFactory.java | 2 +- ...ializer.java => PrimitiveDeserializer.java} | 4 ++-- .../serialize/visitor/ISerializeVisitor.java | 4 ++-- .../serialize/visitor/SerializeVisitor.java | 18 +++++++++--------- .../visitor/array/PrimitiveArrayVisitor.java | 16 ++++------------ 7 files changed, 21 insertions(+), 29 deletions(-) rename native/src/main/java/io/ballerina/lib/avro/deserialize/{GenericDeserializer.java => PrimitiveDeserializer.java} (91%) rename native/src/main/java/io/ballerina/lib/avro/serialize/{GenericSerializer.java => PrimitiveDeserializer.java} (90%) diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java index ee2337b..85e7438 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java @@ -33,7 +33,7 @@ public static Deserializer generateDeserializer(Schema schema, Type type) { case MAP -> new MapDeserializer(schema, type); case RECORD -> new RecordDeserializer(schema, type); case BYTES -> new ByteDeserializer(); - default -> new GenericDeserializer(schema, type); + default -> new PrimitiveDeserializer(schema, type); }; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java similarity index 91% rename from native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java rename to native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java index c85ea70..0cff42e 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/GenericDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java @@ -23,9 +23,9 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; -public class GenericDeserializer extends Deserializer { +public class PrimitiveDeserializer extends Deserializer { - public GenericDeserializer(Schema schema, Type type) { + public PrimitiveDeserializer(Schema schema, Type type) { super(schema, type); } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java index e1814fa..596eb26 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java @@ -32,7 +32,7 @@ public static Serializer createMessage(Schema schema) { case MAP -> new MapSerializer(schema); case RECORD -> new RecordSerializer(schema); case BYTES -> new ByteSerializer(); - default -> new GenericSerializer(schema); + default -> new PrimitiveDeserializer(schema); }; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveDeserializer.java similarity index 90% rename from native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java rename to native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveDeserializer.java index d3fc21d..f6bc65a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/GenericSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveDeserializer.java @@ -21,9 +21,9 @@ import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import org.apache.avro.Schema; -public class GenericSerializer extends Serializer { +public class PrimitiveDeserializer extends Serializer { - public GenericSerializer(Schema schema) { + public PrimitiveDeserializer(Schema schema) { super(schema); } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java index ae26d9a..0290c99 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java @@ -21,7 +21,7 @@ import io.ballerina.lib.avro.serialize.ArraySerializer; import io.ballerina.lib.avro.serialize.EnumSerializer; import io.ballerina.lib.avro.serialize.FixedSerializer; -import io.ballerina.lib.avro.serialize.GenericSerializer; +import io.ballerina.lib.avro.serialize.PrimitiveDeserializer; import io.ballerina.lib.avro.serialize.RecordSerializer; import io.ballerina.runtime.api.values.BArray; import io.ballerina.runtime.api.values.BMap; @@ -35,5 +35,5 @@ public interface ISerializeVisitor { GenericData.Array visit(ArraySerializer arraySerializer, BArray data); Object visit(EnumSerializer enumSerializer, Object data); GenericData.Fixed visit(FixedSerializer fixedSerializer, Object data); - Object visit(GenericSerializer genericSerializer, Object data); + Object visit(PrimitiveDeserializer primitiveDeserializer, Object data); } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java index 2e32b48..a13acc5 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java @@ -21,8 +21,8 @@ import io.ballerina.lib.avro.serialize.ArraySerializer; import io.ballerina.lib.avro.serialize.EnumSerializer; import io.ballerina.lib.avro.serialize.FixedSerializer; -import io.ballerina.lib.avro.serialize.GenericSerializer; import io.ballerina.lib.avro.serialize.MapSerializer; +import io.ballerina.lib.avro.serialize.PrimitiveDeserializer; import io.ballerina.lib.avro.serialize.RecordSerializer; import io.ballerina.lib.avro.serialize.Serializer; import io.ballerina.lib.avro.serialize.UnionSerializer; @@ -56,7 +56,7 @@ public class SerializeVisitor implements ISerializeVisitor { public Serializer createSerializer(Schema schema) { return switch (schema.getValueType().getType()) { case INT, LONG, FLOAT, DOUBLE, BOOLEAN, STRING, BYTES -> - new GenericSerializer(schema.getValueType()); + new PrimitiveDeserializer(schema.getValueType()); case RECORD -> new RecordSerializer(schema.getValueType()); case MAP -> @@ -101,13 +101,13 @@ private Object serializeField(Schema schema, Object fieldData) throws Exception case UNION -> new UnionSerializer(schema).convert(this, fieldData); default -> - new GenericSerializer(schema).convert(this, fieldData); + new PrimitiveDeserializer(schema).convert(this, fieldData); }; } @Override - public Object visit(GenericSerializer genericSerializer, Object data) { - switch (genericSerializer.getSchema().getType()) { + public Object visit(PrimitiveDeserializer primitiveDeserializer, Object data) { + switch (primitiveDeserializer.getSchema().getType()) { case INT -> { return ((Long) data).intValue(); } @@ -171,13 +171,13 @@ public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Ex .filter(type -> type.getType().equals(Schema.Type.ENUM)) .findFirst() .map(type -> visit(new EnumSerializer(type), data)) - .orElse(visit(new GenericSerializer(fieldSchema), data.toString())); + .orElse(visit(new PrimitiveDeserializer(fieldSchema), data.toString())); } case ARRAY_TYPE -> { for (Schema schema : fieldSchema.getTypes()) { switch (schema.getType()) { case BYTES -> { - return new GenericSerializer(schema).convert(this, data); + return new PrimitiveDeserializer(schema).convert(this, data); } case FIXED -> { return new FixedSerializer(schema).convert(this, data); @@ -200,7 +200,7 @@ public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Ex return fieldSchema.getTypes().stream() .filter(schema -> schema.getType().equals(Schema.Type.INT)) .findFirst() - .map(schema -> new GenericSerializer(schema).convert(this, data)) + .map(schema -> new PrimitiveDeserializer(schema).convert(this, data)) .orElse(data); } @@ -208,7 +208,7 @@ public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Ex return fieldSchema.getTypes().stream() .filter(schema -> schema.getType().equals(Schema.Type.FLOAT)) .findFirst() - .map(schema -> new GenericSerializer(schema).convert(this, data)) + .map(schema -> new PrimitiveDeserializer(schema).convert(this, data)) .orElse(data); } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java index 399ab75..d3c52c3 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java @@ -16,44 +16,36 @@ public GenericData.Array visit(BArray data, Schema schema, GenericData.A : schema.getType(); switch (type) { - case STRING -> { - array.addAll(Arrays.asList(data.getStringArray())); - return array; - } + case STRING -> + array.addAll(Arrays.asList(data.getStringArray())); case INT -> { for (long obj: data.getIntArray()) { array.add(((Long) obj).intValue()); } - return array; } case LONG -> { for (Object obj: data.getIntArray()) { array.add(obj); } - return array; } case FLOAT -> { for (Double obj: data.getFloatArray()) { array.add(obj.floatValue()); } - return array; } case DOUBLE -> { for (Object obj: data.getFloatArray()) { array.add(obj); } - return array; } case BOOLEAN -> { for (Object obj: data.getBooleanArray()) { array.add(obj); } - return array; - } - default -> { - return visitBytes(data, array); } + default -> visitBytes(data, array); } + return array; } From a4d9890a5d3da6c171efd5414827baf51758c6ca Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 11:30:12 +0530 Subject: [PATCH 28/47] Move utility functions of record APIs to a seperate class --- .../avro/deserialize/ByteDeserializer.java | 1 + .../visitor/DeserializeRecordVisitor.java | 3 +- .../visitor/DeserializeVisitor.java | 139 +++++++----------- .../avro/deserialize/visitor/RecordUtils.java | 64 ++++++++ 4 files changed, 122 insertions(+), 85 deletions(-) create mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java index 7f61a23..43235b6 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java @@ -4,6 +4,7 @@ import org.apache.avro.generic.GenericData; public class ByteDeserializer extends Deserializer { + @Override public Object visit(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visitBytes(data); diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java index 365f254..8ecb1da 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java @@ -16,7 +16,6 @@ import org.apache.avro.generic.GenericRecord; import java.nio.ByteBuffer; -import java.util.Map; import static io.ballerina.lib.avro.Utils.getMutableType; import static io.ballerina.lib.avro.deserialize.visitor.UnionRecordUtils.visitUnionRecords; @@ -69,7 +68,7 @@ private void processMapField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { Type mapType = extractMapType(avroRecord.getType()); MapDeserializer mapDeserializer = new MapDeserializer(field.schema(), mapType); - Object fieldValue = mapDeserializer.visit(this, (Map) fieldData); + Object fieldValue = mapDeserializer.visit(this, fieldData); avroRecord.put(StringUtils.fromString(field.name()), fieldValue); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java index 4d3cd81..3460928 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -22,10 +22,9 @@ import io.ballerina.lib.avro.deserialize.Deserializer; import io.ballerina.lib.avro.deserialize.EnumDeserializer; import io.ballerina.lib.avro.deserialize.FixedDeserializer; -import io.ballerina.lib.avro.deserialize.GenericDeserializer; import io.ballerina.lib.avro.deserialize.MapDeserializer; +import io.ballerina.lib.avro.deserialize.PrimitiveDeserializer; import io.ballerina.lib.avro.deserialize.RecordDeserializer; -import io.ballerina.lib.avro.deserialize.StringDeserializer; import io.ballerina.lib.avro.deserialize.UnionDeserializer; import io.ballerina.runtime.api.TypeTags; import io.ballerina.runtime.api.creators.ValueCreator; @@ -36,6 +35,7 @@ import io.ballerina.runtime.api.types.RecordType; import io.ballerina.runtime.api.types.ReferenceType; import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; import io.ballerina.runtime.api.utils.TypeUtils; import io.ballerina.runtime.api.utils.ValueUtils; import io.ballerina.runtime.api.values.BArray; @@ -59,7 +59,12 @@ import static io.ballerina.lib.avro.Utils.REFERENCE_TYPE; import static io.ballerina.lib.avro.Utils.STRING_TYPE; import static io.ballerina.lib.avro.Utils.getMutableType; -import static io.ballerina.lib.avro.deserialize.visitor.UnionRecordUtils.visitUnionRecords; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processArrayField; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processBytesField; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processMapField; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processRecordField; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processStringField; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processUnionField; import static io.ballerina.runtime.api.utils.StringUtils.fromString; public class DeserializeVisitor implements IDeserializeVisitor { @@ -77,7 +82,7 @@ public static Deserializer createDeserializer(Schema schema, Type type) { case FIXED -> new FixedDeserializer(schema, type); default -> - new GenericDeserializer(schema, type); + new PrimitiveDeserializer(schema, type); }; } @@ -128,29 +133,30 @@ public BMap visit(MapDeserializer mapDeserializer, Map processMapArray(avroRecord, schema, (MapType) type, key, (GenericData.Array) value); case BYTES -> - avroRecord.put(fromString(key.toString()), - ValueCreator.createArrayValue(((ByteBuffer) value).array())); + avroRecord.put(StringUtils.fromString(key.toString()), + ValueCreator.createArrayValue(((ByteBuffer) value).array())); case FIXED -> - avroRecord.put(fromString(key.toString()), - ValueCreator.createArrayValue(((GenericFixed) value).bytes())); + avroRecord.put(StringUtils.fromString(key.toString()), + ValueCreator.createArrayValue(((GenericFixed) value).bytes())); case ENUM, STRING -> - avroRecord.put(fromString(key.toString()), - fromString(value.toString())); + avroRecord.put(StringUtils.fromString(key.toString()), + StringUtils.fromString(value.toString())); case RECORD -> processMapRecord(avroRecord, schema, (MapType) type, key, (GenericRecord) value); case FLOAT -> - avroRecord.put(fromString(key.toString()), Double.parseDouble(value.toString())); + avroRecord.put(StringUtils.fromString(key.toString()), + Double.parseDouble(value.toString())); case MAP -> processMaps(avroRecord, schema, (MapType) type, key, (Map) value); default -> - avroRecord.put(fromString(key.toString()), value); + avroRecord.put(StringUtils.fromString(key.toString()), value); } } return (BMap) ValueUtils.convert(avroRecord, type); } - public Object visit(GenericDeserializer genericDeserializer, Object data) { - Schema schema = genericDeserializer.getSchema(); + public Object visit(PrimitiveDeserializer primitiveDeserializer, Object data) { + Schema schema = primitiveDeserializer.getSchema(); if (schema.getType().equals(Schema.Type.ARRAY)) { GenericData.Array array = (GenericData.Array) data; switch (schema.getElementType().getType()) { @@ -170,7 +176,7 @@ public Object visit(GenericDeserializer genericDeserializer, Object data) { return visitBooleanArray(array); } default -> { - return visitBytesArray(array, genericDeserializer.getType()); + return visitBytesArray(array, primitiveDeserializer.getType()); } } } else { @@ -195,20 +201,10 @@ public BArray visit(UnionDeserializer unionDeserializer, GenericData.Array { - RecordDeserializer recordDeserializer = new RecordDeserializer(schema.getElementType(), type); - return (BArray) recordDeserializer.visit(this, data); + return visitRecordArray(data, type, schema); } case ARRAY_TYPE -> { - Object[] objects = new Object[data.size()]; - Type elementType = ((ArrayType) type).getElementType(); - ArrayDeserializer arrayDeserializer = new ArrayDeserializer(schema.getElementType(), elementType); - int index = 0; - for (Object currentData : data) { - Object deserializedObject = arrayDeserializer.visit(this, (GenericData.Array) currentData); - objects[index++] = deserializedObject; - } - return ValueCreator.createArrayValue(objects, (ArrayType) type); - + return visitUnionArray(data, (ArrayType) type, schema); } default -> { return visitBytes(data); @@ -216,6 +212,23 @@ public BArray visit(UnionDeserializer unionDeserializer, GenericData.Array data, Type type, Schema schema) throws Exception { + RecordDeserializer recordDeserializer = new RecordDeserializer(schema.getElementType(), type); + return (BArray) recordDeserializer.visit(this, data); + } + + private BArray visitUnionArray(GenericData.Array data, ArrayType type, Schema schema) throws Exception { + Object[] objects = new Object[data.size()]; + Type elementType = type.getElementType(); + ArrayDeserializer arrayDeserializer = new ArrayDeserializer(schema.getElementType(), elementType); + int index = 0; + for (Object currentData : data) { + Object deserializedObject = arrayDeserializer.visit(this, (GenericData.Array) currentData); + objects[index++] = deserializedObject; + } + return ValueCreator.createArrayValue(objects, type); + } + public BArray visit(RecordDeserializer recordDeserializer, GenericData.Array data) throws Exception { List recordList = new ArrayList<>(); Type type = recordDeserializer.getType(); @@ -247,47 +260,6 @@ private BMap createAvroRecord(Type type) { return ValueCreator.createRecordValue((RecordType) type); } - private void processMapField(BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - Type mapType = extractMapType(avroRecord.getType()); - MapDeserializer mapDeserializer = new MapDeserializer(field.schema(), mapType); - Object fieldValue = mapDeserializer.visit(this, (Map) fieldData); - avroRecord.put(fromString(field.name()), fieldValue); - } - - private void processArrayField(BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - ArrayDeserializer arrayDes = new ArrayDeserializer(field.schema(), avroRecord.getType()); - Object fieldValue = arrayDes.visit(this, (GenericData.Array) fieldData); - avroRecord.put(fromString(field.name()), fieldValue); - } - - private void processBytesField(BMap avroRecord, Schema.Field field, Object fieldData) { - ByteBuffer byteBuffer = (ByteBuffer) fieldData; - Object fieldValue = ValueCreator.createArrayValue(byteBuffer.array()); - avroRecord.put(fromString(field.name()), fieldValue); - } - - private void processRecordField(BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - Type recType = extractRecordType((RecordType) avroRecord.getType()); - RecordDeserializer recordDes = new RecordDeserializer(field.schema(), recType); - Object fieldValue = recordDes.visit(this, (GenericRecord) fieldData); - avroRecord.put(fromString(field.name()), fieldValue); - } - - private void processStringField(BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - StringDeserializer stringDes = new StringDeserializer(); - Object fieldValue = stringDes.visit(this, fieldData); - avroRecord.put(fromString(field.name()), fieldValue); - } - - private void processUnionField(Type type, BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - visitUnionRecords(type, avroRecord, field, fieldData); - } - private void processMaps(BMap avroRecord, Schema schema, MapType type, Object key, Map value) throws Exception { Schema fieldSchema = schema.getValueType(); @@ -431,29 +403,31 @@ public BString visitString(Object data) { return fromString(data.toString()); } - public static Type extractMapType(Type type) { + public static Type extractMapType(Type type) throws Exception { Type mapType = type; assert type instanceof RecordType; + if (type.getTag() != TypeTags.RECORD_TYPE_TAG) { + throw new Exception("Type is not a record type."); + } for (Map.Entry entry : ((RecordType) type).getFields().entrySet()) { Field fieldValue = entry.getValue(); if (fieldValue != null) { Type fieldType = fieldValue.getFieldType(); switch (fieldType.getTag()) { - case TypeTags.MAP_TAG: - mapType = fieldType; - break; - case TypeTags.INTERSECTION_TAG: + case TypeTags.MAP_TAG -> + mapType = fieldType; + case TypeTags.INTERSECTION_TAG -> { Type referredType = getMutableType((IntersectionType) fieldType); if (referredType.getTag() == TypeTags.MAP_TAG) { mapType = referredType; } - break; - default: + } + default -> { Type referType = TypeUtils.getReferredType(fieldType); if (referType.getTag() == TypeTags.MAP_TAG) { mapType = referType; } - break; + } } } } @@ -468,21 +442,20 @@ public static RecordType extractRecordType(RecordType type) { if (fieldValue != null) { Type fieldType = fieldValue.getFieldType(); switch (fieldType.getTag()) { - case TypeTags.RECORD_TYPE_TAG: - recType = (RecordType) fieldType; - break; - case TypeTags.INTERSECTION_TAG: + case TypeTags.RECORD_TYPE_TAG -> + recType = (RecordType) fieldType; + case TypeTags.INTERSECTION_TAG -> { Type getType = getMutableType((IntersectionType) fieldType); if (getType.getTag() == TypeTags.RECORD_TYPE_TAG) { recType = (RecordType) getType; } - break; - default: + } + default -> { Type referredType = TypeUtils.getReferredType(fieldType); if (referredType.getTag() == TypeTags.RECORD_TYPE_TAG) { recType = (RecordType) referredType; } - break; + } } } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java new file mode 100644 index 0000000..ff0d9b5 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java @@ -0,0 +1,64 @@ +package io.ballerina.lib.avro.deserialize.visitor; + +import io.ballerina.lib.avro.deserialize.ArrayDeserializer; +import io.ballerina.lib.avro.deserialize.MapDeserializer; +import io.ballerina.lib.avro.deserialize.RecordDeserializer; +import io.ballerina.lib.avro.deserialize.StringDeserializer; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.nio.ByteBuffer; + +import static io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor.extractMapType; +import static io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor.extractRecordType; +import static io.ballerina.lib.avro.deserialize.visitor.UnionRecordUtils.visitUnionRecords; +import static io.ballerina.runtime.api.utils.StringUtils.fromString; + +public class RecordUtils { + + public static void processMapField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + Type mapType = extractMapType(avroRecord.getType()); + MapDeserializer mapDeserializer = new MapDeserializer(field.schema(), mapType); + Object fieldValue = mapDeserializer.visit(new DeserializeVisitor(), fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + public static void processArrayField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + ArrayDeserializer arrayDes = new ArrayDeserializer(field.schema(), avroRecord.getType()); + Object fieldValue = arrayDes.visit(new DeserializeVisitor(), (GenericData.Array) fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + public static void processBytesField(BMap avroRecord, Schema.Field field, Object fieldData) { + ByteBuffer byteBuffer = (ByteBuffer) fieldData; + Object fieldValue = ValueCreator.createArrayValue(byteBuffer.array()); + avroRecord.put(fromString(field.name()), fieldValue); + } + + public static void processRecordField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + Type recType = extractRecordType((RecordType) avroRecord.getType()); + RecordDeserializer recordDes = new RecordDeserializer(field.schema(), recType); + Object fieldValue = recordDes.visit(new DeserializeVisitor(), fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + public static void processStringField(BMap avroRecord, + Schema.Field field, Object fieldData) { + StringDeserializer stringDes = new StringDeserializer(); + Object fieldValue = stringDes.visit(new DeserializeVisitor(), fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + public static void processUnionField(Type type, BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + visitUnionRecords(type, avroRecord, field, fieldData); + } +} From 67082dbb77efeb553a548fdd215d38db394a43d6 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 11:55:35 +0530 Subject: [PATCH 29/47] Fix review comments --- .../lib/avro/deserialize/ArrayDeserializer.java | 2 +- .../ballerina/lib/avro/deserialize/Deserializer.java | 12 +++++------- .../lib/avro/deserialize/FixedDeserializer.java | 2 +- .../lib/avro/deserialize/MapDeserializer.java | 2 +- .../lib/avro/deserialize/PrimitiveDeserializer.java | 2 +- .../lib/avro/deserialize/RecordDeserializer.java | 2 +- .../lib/avro/deserialize/UnionDeserializer.java | 2 +- 7 files changed, 11 insertions(+), 13 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java index 400554d..5a1872a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java @@ -26,7 +26,7 @@ public class ArrayDeserializer extends Deserializer { public ArrayDeserializer(Schema schema, Type type) { - super(schema, type); + super(type, schema); } @Override diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java index efe498c..744bb4a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java @@ -30,18 +30,16 @@ public abstract class Deserializer { private final Type type; public Deserializer() { - this.schema = null; - this.type = null; + this(null); } public Deserializer(Type type) { - this.schema = null; - this.type = TypeUtils.getReferredType(type); + this(type, null); } - public Deserializer(Schema schema, Type type) { - this.schema = new Schema.Parser().parse(schema.toString()); - this.type = TypeUtils.getReferredType(type); + public Deserializer(Type type, Schema schema) { + this.schema = schema == null ? null : new Schema.Parser().parse(schema.toString()); + this.type = type == null ? null : TypeUtils.getReferredType(type); } public Schema getSchema() { diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java index 208ff42..cf22612 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java @@ -26,7 +26,7 @@ public class FixedDeserializer extends Deserializer { public FixedDeserializer(Schema schema, Type type) { - super(schema, type); + super(type, schema); } @Override diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java index b505327..0f01ea4 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java @@ -28,7 +28,7 @@ public class MapDeserializer extends Deserializer { public MapDeserializer(Schema schema, Type type) { - super(schema, type); + super(type, schema); } @Override diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java index 0cff42e..555456f 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java @@ -26,7 +26,7 @@ public class PrimitiveDeserializer extends Deserializer { public PrimitiveDeserializer(Schema schema, Type type) { - super(schema, type); + super(type, schema); } @Override diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java index 2f0d842..aa48398 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java @@ -27,7 +27,7 @@ public class RecordDeserializer extends Deserializer { public RecordDeserializer(Schema schema, Type type) { - super(schema, type); + super(type, schema); } @Override diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java index 51f8148..d2d1420 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java @@ -8,7 +8,7 @@ public class UnionDeserializer extends Deserializer { public UnionDeserializer(Schema schema, Type type) { - super(schema, type); + super(type, schema); } @Override From 64e73f33abd5c4bf6e13d412e09fe6fdf68ee010 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 13:22:55 +0530 Subject: [PATCH 30/47] Exclude unconfirmed cast bug by using type tags for type confirmation --- build-config/spotbugs-exclude.xml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 build-config/spotbugs-exclude.xml diff --git a/build-config/spotbugs-exclude.xml b/build-config/spotbugs-exclude.xml new file mode 100644 index 0000000..2079eda --- /dev/null +++ b/build-config/spotbugs-exclude.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + From e09ff1b7ca0c9d33c91d85a1f8c890e8534fb8ab Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 13:23:14 +0530 Subject: [PATCH 31/47] Remove unnecessary type tag declarations --- .../java/io/ballerina/lib/avro/Utils.java | 36 ++++++------ .../visitor/DeserializeRecordVisitor.java | 10 +--- .../visitor/DeserializeVisitor.java | 32 ++++------- .../serialize/visitor/SerializeVisitor.java | 57 +++++++++---------- 4 files changed, 54 insertions(+), 81 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/Utils.java b/native/src/main/java/io/ballerina/lib/avro/Utils.java index 3a462af..fbd4039 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Utils.java +++ b/native/src/main/java/io/ballerina/lib/avro/Utils.java @@ -38,35 +38,31 @@ private Utils() { public static final String ERROR_TYPE = "Error"; public static final String SERIALIZATION_ERROR = "Avro serialization error"; public static final String DESERIALIZATION_ERROR = "Avro deserialization error"; - public static final int STRING_TYPE = 5; - public static final int ARRAY_TYPE = 32; - public static final int MAP_TYPE = 27; - public static final int INTERSECTION_TYPE = 34; - public static final int REFERENCE_TYPE = 53; - public static final int RECORD_TYPE = 24; - public static final int INTEGER_TYPE = 1; - public static final int FLOAT_TYPE = 3; - public static final int BOOLEAN_TYPE = 6; public static BError createError(String message, Throwable throwable) { BError cause = ErrorCreator.createError(throwable); return ErrorCreator.createError(getModule(), ERROR_TYPE, StringUtils.fromString(message), cause, null); } - public static Type getMutableType(IntersectionType intersectionType) { - for (Type type : intersectionType.getConstituentTypes()) { - Type referredType = TypeUtils.getImpliedType(type); - if (referredType.getTag() == TypeTags.UNION_TAG) { - for (Type elementType : ((UnionType) referredType).getMemberTypes()) { - if (elementType.getTag() == TypeTags.MAP_TAG) { - return elementType; + public static Type getMutableType(Type dataType) { + if (dataType.getTag() == TypeTags.INTERSECTION_TAG) { + IntersectionType intersectionType = (IntersectionType) dataType; + for (Type type : intersectionType.getConstituentTypes()) { + Type referredType = TypeUtils.getImpliedType(type); + if (referredType.getTag() == TypeTags.UNION_TAG) { + for (Type elementType : ((UnionType) referredType).getMemberTypes()) { + if (elementType.getTag() == TypeTags.MAP_TAG) { + return elementType; + } } } + if (TypeUtils.getImpliedType(intersectionType.getEffectiveType()).getTag() == referredType.getTag()) { + return referredType; + } } - if (TypeUtils.getImpliedType(intersectionType.getEffectiveType()).getTag() == referredType.getTag()) { - return referredType; - } + } else { + return dataType; } - throw new IllegalStateException("Unsupported intersection type found: " + intersectionType); + throw new IllegalStateException("Unsupported intersection type found."); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java index 8ecb1da..a8bb9f6 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java @@ -5,7 +5,6 @@ import io.ballerina.lib.avro.deserialize.RecordDeserializer; import io.ballerina.lib.avro.deserialize.StringDeserializer; import io.ballerina.runtime.api.creators.ValueCreator; -import io.ballerina.runtime.api.types.IntersectionType; import io.ballerina.runtime.api.types.RecordType; import io.ballerina.runtime.api.types.Type; import io.ballerina.runtime.api.utils.StringUtils; @@ -26,7 +25,7 @@ public BMap visit(RecordDeserializer recordDeserializer, Generi Type originalType = recordDeserializer.getType(); Type type = recordDeserializer.getType(); Schema schema = recordDeserializer.getSchema(); - BMap avroRecord = createAvroRecord(type); + BMap avroRecord = ValueCreator.createRecordValue((RecordType) getMutableType(type)); for (Schema.Field field : schema.getFields()) { Object fieldData = rec.get(field.name()); switch (field.schema().getType()) { @@ -57,13 +56,6 @@ public BMap visit(RecordDeserializer recordDeserializer, Generi return avroRecord; } - private BMap createAvroRecord(Type type) { - if (type instanceof IntersectionType) { - type = getMutableType((IntersectionType) type); - } - return ValueCreator.createRecordValue((RecordType) type); - } - private void processMapField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { Type mapType = extractMapType(avroRecord.getType()); diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java index 3460928..a7ee89b 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -51,13 +51,6 @@ import java.util.List; import java.util.Map; -import static io.ballerina.lib.avro.Utils.ARRAY_TYPE; -import static io.ballerina.lib.avro.Utils.BOOLEAN_TYPE; -import static io.ballerina.lib.avro.Utils.FLOAT_TYPE; -import static io.ballerina.lib.avro.Utils.INTEGER_TYPE; -import static io.ballerina.lib.avro.Utils.RECORD_TYPE; -import static io.ballerina.lib.avro.Utils.REFERENCE_TYPE; -import static io.ballerina.lib.avro.Utils.STRING_TYPE; import static io.ballerina.lib.avro.Utils.getMutableType; import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processArrayField; import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processBytesField; @@ -188,22 +181,22 @@ public BArray visit(UnionDeserializer unionDeserializer, GenericData.Array { + case TypeTags.STRING_TAG -> { return visitStringArray(data); } - case FLOAT_TYPE -> { + case TypeTags.FLOAT_TAG -> { return visitDoubleArray(data); } - case BOOLEAN_TYPE -> { + case TypeTags.BOOLEAN_TAG -> { return visitBooleanArray(data); } - case INTEGER_TYPE -> { + case TypeTags.INT_TAG -> { return visitIntegerArray(data, schema); } - case RECORD_TYPE -> { + case TypeTags.RECORD_TYPE_TAG -> { return visitRecordArray(data, type, schema); } - case ARRAY_TYPE -> { + case TypeTags.ARRAY_TAG -> { return visitUnionArray(data, (ArrayType) type, schema); } default -> { @@ -234,14 +227,14 @@ public BArray visit(RecordDeserializer recordDeserializer, GenericData.Array { + case TypeTags.ARRAY_TAG -> { for (Object datum : data) { Type fieldType = ((ArrayType) type).getElementType().getCachedReferredType(); RecordDeserializer recordDes = new RecordDeserializer(schema.getElementType(), fieldType); recordList.add(recordDes.visit(this, (GenericRecord) datum)); } } - case REFERENCE_TYPE -> { + case TypeTags.TYPE_REFERENCED_TYPE_TAG -> { for (Object datum : data) { Type fieldType = ((ReferenceType) type).getReferredType(); RecordDeserializer recordDes = new RecordDeserializer(schema.getElementType(), fieldType); @@ -249,15 +242,11 @@ public BArray visit(RecordDeserializer recordDeserializer, GenericData.Array createAvroRecord(Type type) { - if (type instanceof IntersectionType) { - type = getMutableType((IntersectionType) type); - } - return ValueCreator.createRecordValue((RecordType) type); + return ValueCreator.createRecordValue((RecordType) getMutableType(type)); } private void processMaps(BMap avroRecord, Schema schema, @@ -405,7 +394,6 @@ public BString visitString(Object data) { public static Type extractMapType(Type type) throws Exception { Type mapType = type; - assert type instanceof RecordType; if (type.getTag() != TypeTags.RECORD_TYPE_TAG) { throw new Exception("Type is not a record type."); } @@ -445,7 +433,7 @@ public static RecordType extractRecordType(RecordType type) { case TypeTags.RECORD_TYPE_TAG -> recType = (RecordType) fieldType; case TypeTags.INTERSECTION_TAG -> { - Type getType = getMutableType((IntersectionType) fieldType); + Type getType = getMutableType(fieldType); if (getType.getTag() == TypeTags.RECORD_TYPE_TAG) { recType = (RecordType) getType; } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java index a13acc5..99a7ed2 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java @@ -28,6 +28,7 @@ import io.ballerina.lib.avro.serialize.UnionSerializer; import io.ballerina.lib.avro.serialize.visitor.array.ArrayVisitorFactory; import io.ballerina.lib.avro.serialize.visitor.array.IArrayVisitor; +import io.ballerina.runtime.api.TypeTags; import io.ballerina.runtime.api.types.Type; import io.ballerina.runtime.api.utils.StringUtils; import io.ballerina.runtime.api.utils.TypeUtils; @@ -44,13 +45,6 @@ import java.util.Map; import java.util.Objects; -import static io.ballerina.lib.avro.Utils.ARRAY_TYPE; -import static io.ballerina.lib.avro.Utils.FLOAT_TYPE; -import static io.ballerina.lib.avro.Utils.INTEGER_TYPE; -import static io.ballerina.lib.avro.Utils.MAP_TYPE; -import static io.ballerina.lib.avro.Utils.RECORD_TYPE; -import static io.ballerina.lib.avro.Utils.STRING_TYPE; - public class SerializeVisitor implements ISerializeVisitor { public Serializer createSerializer(Schema schema) { @@ -166,37 +160,24 @@ public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Ex Schema fieldSchema = unionSerializer.getSchema(); Type typeName = TypeUtils.getType(data); switch (typeName.getTag()) { - case STRING_TYPE -> { + case TypeTags.STRING_TAG -> { return fieldSchema.getTypes().stream() .filter(type -> type.getType().equals(Schema.Type.ENUM)) .findFirst() .map(type -> visit(new EnumSerializer(type), data)) .orElse(visit(new PrimitiveDeserializer(fieldSchema), data.toString())); } - case ARRAY_TYPE -> { - for (Schema schema : fieldSchema.getTypes()) { - switch (schema.getType()) { - case BYTES -> { - return new PrimitiveDeserializer(schema).convert(this, data); - } - case FIXED -> { - return new FixedSerializer(schema).convert(this, data); - } - case ARRAY -> { - return new ArraySerializer(schema).convert(this, data); - } - } - } - return new ArraySerializer(fieldSchema).convert(this, data); + case TypeTags.ARRAY_TAG -> { + return visitUnionArrays(data, fieldSchema); } - case MAP_TYPE -> { + case TypeTags.MAP_TAG -> { return new MapSerializer(fieldSchema).convert(this, data); } - case RECORD_TYPE -> { - return new RecordSerializer(getRecordSchema(Schema.Type.RECORD, fieldSchema.getTypes())) - .convert(this, data); + case TypeTags.RECORD_TYPE_TAG -> { + return new RecordSerializer(getRecordSchema(Schema.Type.RECORD, + fieldSchema.getTypes())).convert(this, data); } - case INTEGER_TYPE -> { + case TypeTags.INT_TAG -> { return fieldSchema.getTypes().stream() .filter(schema -> schema.getType().equals(Schema.Type.INT)) .findFirst() @@ -204,13 +185,12 @@ public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Ex .orElse(data); } - case FLOAT_TYPE -> { + case TypeTags.FLOAT_TAG -> { return fieldSchema.getTypes().stream() .filter(schema -> schema.getType().equals(Schema.Type.FLOAT)) .findFirst() .map(schema -> new PrimitiveDeserializer(schema).convert(this, data)) .orElse(data); - } default -> { return data; @@ -218,6 +198,23 @@ public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Ex } } + private Object visitUnionArrays(Object data, Schema fieldSchema) { + for (Schema schema : fieldSchema.getTypes()) { + switch (schema.getType()) { + case BYTES -> { + return new PrimitiveDeserializer(schema).convert(this, data); + } + case FIXED -> { + return new FixedSerializer(schema).convert(this, data); + } + case ARRAY -> { + return new ArraySerializer(schema).convert(this, data); + } + } + } + return new ArraySerializer(fieldSchema).convert(this, data); + } + public static Schema getRecordSchema(Schema.Type givenType, List schemas) { for (Schema schema: schemas) { if (schema.getType().equals(Schema.Type.UNION)) { From e2f90c016266b1f5e1f80427a02b9ab608844401 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 13:26:48 +0530 Subject: [PATCH 32/47] Update serializing APIs --- .../serialize/visitor/SerializeVisitor.java | 47 ++++++++++++------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java index 99a7ed2..2026e71 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java @@ -161,11 +161,7 @@ public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Ex Type typeName = TypeUtils.getType(data); switch (typeName.getTag()) { case TypeTags.STRING_TAG -> { - return fieldSchema.getTypes().stream() - .filter(type -> type.getType().equals(Schema.Type.ENUM)) - .findFirst() - .map(type -> visit(new EnumSerializer(type), data)) - .orElse(visit(new PrimitiveDeserializer(fieldSchema), data.toString())); + return visitiUnionStrings(data, fieldSchema); } case TypeTags.ARRAY_TAG -> { return visitUnionArrays(data, fieldSchema); @@ -174,23 +170,14 @@ public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Ex return new MapSerializer(fieldSchema).convert(this, data); } case TypeTags.RECORD_TYPE_TAG -> { - return new RecordSerializer(getRecordSchema(Schema.Type.RECORD, - fieldSchema.getTypes())).convert(this, data); + Schema schema = getRecordSchema(Schema.Type.RECORD, fieldSchema.getTypes()); + return new RecordSerializer(schema).convert(this, data); } case TypeTags.INT_TAG -> { - return fieldSchema.getTypes().stream() - .filter(schema -> schema.getType().equals(Schema.Type.INT)) - .findFirst() - .map(schema -> new PrimitiveDeserializer(schema).convert(this, data)) - .orElse(data); - + return visitUnionIntegers(data, fieldSchema); } case TypeTags.FLOAT_TAG -> { - return fieldSchema.getTypes().stream() - .filter(schema -> schema.getType().equals(Schema.Type.FLOAT)) - .findFirst() - .map(schema -> new PrimitiveDeserializer(schema).convert(this, data)) - .orElse(data); + return visitUnionFloats(data, fieldSchema); } default -> { return data; @@ -198,6 +185,30 @@ public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Ex } } + private Object visitUnionFloats(Object data, Schema fieldSchema) { + return fieldSchema.getTypes().stream() + .filter(schema -> schema.getType().equals(Schema.Type.FLOAT)) + .findFirst() + .map(schema -> new PrimitiveDeserializer(schema).convert(this, data)) + .orElse(data); + } + + private Object visitUnionIntegers(Object data, Schema fieldSchema) { + return fieldSchema.getTypes().stream() + .filter(schema -> schema.getType().equals(Schema.Type.INT)) + .findFirst() + .map(schema -> new PrimitiveDeserializer(schema).convert(this, data)) + .orElse(data); + } + + private Object visitiUnionStrings(Object data, Schema fieldSchema) { + return fieldSchema.getTypes().stream() + .filter(type -> type.getType().equals(Schema.Type.ENUM)) + .findFirst() + .map(type -> visit(new EnumSerializer(type), data)) + .orElse(visit(new PrimitiveDeserializer(fieldSchema), data.toString())); + } + private Object visitUnionArrays(Object data, Schema fieldSchema) { for (Schema schema : fieldSchema.getTypes()) { switch (schema.getType()) { From 9ee7488b67ebfa097f633e79db21a0248d617d75 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 14:26:17 +0530 Subject: [PATCH 33/47] Add license headers --- .../lib/avro/deserialize/ByteDeserializer.java | 18 ++++++++++++++++++ .../lib/avro/deserialize/EnumDeserializer.java | 18 ++++++++++++++++++ .../avro/deserialize/UnionDeserializer.java | 18 ++++++++++++++++++ .../visitor/DeserializeArrayVisitor.java | 18 ++++++++++++++++++ .../visitor/DeserializeRecordVisitor.java | 18 ++++++++++++++++++ .../avro/deserialize/visitor/RecordUtils.java | 18 ++++++++++++++++++ .../deserialize/visitor/UnionRecordUtils.java | 18 ++++++++++++++++++ .../lib/avro/serialize/ByteSerializer.java | 18 ++++++++++++++++++ .../lib/avro/serialize/Serializer.java | 18 ++++++++++++++++++ .../lib/avro/serialize/UnionSerializer.java | 18 ++++++++++++++++++ .../serialize/visitor/array/ArrayVisitor.java | 18 ++++++++++++++++++ .../visitor/array/ArrayVisitorFactory.java | 18 ++++++++++++++++++ .../visitor/array/EnumArrayVisitor.java | 18 ++++++++++++++++++ .../visitor/array/FixedArrayVisitor.java | 18 ++++++++++++++++++ .../serialize/visitor/array/IArrayVisitor.java | 18 ++++++++++++++++++ .../visitor/array/MapArrayVisitor.java | 18 ++++++++++++++++++ .../visitor/array/PrimitiveArrayVisitor.java | 18 ++++++++++++++++++ .../visitor/array/RecordArrayVisitor.java | 18 ++++++++++++++++++ .../visitor/array/UnionArrayVisitor.java | 18 ++++++++++++++++++ 19 files changed, 342 insertions(+) diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java index 43235b6..0c51188 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.deserialize; import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java index c0c06c3..f4770de 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.deserialize; import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java index d2d1420..980536b 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.deserialize; import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java index 26ea4cf..ec85357 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.deserialize.visitor; import io.ballerina.lib.avro.deserialize.ArrayDeserializer; diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java index a8bb9f6..5ff489f 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.deserialize.visitor; import io.ballerina.lib.avro.deserialize.ArrayDeserializer; diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java index ff0d9b5..c219efb 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.deserialize.visitor; import io.ballerina.lib.avro.deserialize.ArrayDeserializer; diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java index 664cea7..0f98feb 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.deserialize.visitor; import io.ballerina.lib.avro.deserialize.RecordDeserializer; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java index 705d649..c24905b 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize; import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java index 309945a..67fa6ae 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize; import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java index fbe4bc5..0807a7b 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize; import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java index fd9afbe..db6aeaf 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize.visitor.array; import io.ballerina.lib.avro.serialize.ArraySerializer; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java index 36a9634..14045b4 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize.visitor.array; import org.apache.avro.Schema; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java index 5757ac4..9076336 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize.visitor.array; import io.ballerina.runtime.api.values.BArray; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java index 9a401dc..fec45c6 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize.visitor.array; import io.ballerina.runtime.api.values.BArray; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java index 2645378..e126a83 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize.visitor.array; import io.ballerina.runtime.api.values.BArray; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java index 1674e96..6c169db 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize.visitor.array; import io.ballerina.lib.avro.serialize.MapSerializer; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java index d3c52c3..c1dc886 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize.visitor.array; import io.ballerina.runtime.api.values.BArray; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java index 93ff3d0..5e4a71a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize.visitor.array; import io.ballerina.lib.avro.serialize.RecordSerializer; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java index 65b1280..7d7e48c 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java @@ -1,3 +1,21 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package io.ballerina.lib.avro.serialize.visitor.array; import io.ballerina.runtime.api.values.BArray; From 29cb2fc482d9965325e3b968f3c6746305554e52 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 21:57:12 +0530 Subject: [PATCH 34/47] Fix readonly data support --- ballerina/tests/array_tests.bal | 232 ++++++-- ballerina/tests/byte_tests.bal | 50 +- ballerina/tests/map_tests.bal | 515 ++++++++++++++---- ballerina/tests/primitive_tests.bal | 46 +- ballerina/tests/record_tests.bal | 53 +- ballerina/tests/test.bal | 253 ++++++++- ballerina/tests/types.bal | 5 + .../visitor/DeserializeArrayVisitor.java | 15 +- .../visitor/DeserializeVisitor.java | 48 +- 9 files changed, 964 insertions(+), 253 deletions(-) diff --git a/ballerina/tests/array_tests.bal b/ballerina/tests/array_tests.bal index 3cd3a08..07a0a83 100644 --- a/ballerina/tests/array_tests.bal +++ b/ballerina/tests/array_tests.bal @@ -31,9 +31,29 @@ public isolated function testIntArrays() returns error? { int[] numbers = [22, 556, 78]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(numbers); - int[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, numbers); + byte[] encodedValue = check avro.toAvro(numbers); + int[] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, numbers); +} + +@test:Config { + groups: ["array", "int", "qqq"] +} +public isolated function testReadOnlyIntArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "integers", + "namespace": "data", + "items": "int" + }`; + + int[] & readonly numbers = [22, 556, 78]; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(numbers); + int[] & readonly deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, numbers); } @test:Config { @@ -51,9 +71,29 @@ public isolated function testStringArrays() returns error? { string[] colors = ["red", "green", "blue"]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - string[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, colors); + byte[] encodedValue = check avro.toAvro(colors); + string[] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, colors); +} + +@test:Config { + groups: ["array", "string"] +} +public isolated function testReadOnlyStringArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "stringArray", + "namespace": "data", + "items": "string" + }`; + + string[] & readonly colors = ["red", "green", "blue"]; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(colors); + string[] & readonly deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -76,9 +116,34 @@ public isolated function testArrayOfStringArrays() returns error? { string[][] colors = [["red", "green", "blue"]]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - string[][] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, colors); + byte[] encodedValue = check avro.toAvro(colors); + string[][] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, colors); +} + +@test:Config { + groups: ["array", "string", "mn"] +} +public isolated function testReadOnlyArrayOfStringArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "stringArray", + "namespace": "data", + "items": { + "type": "array", + "name" : "strings", + "namespace": "data", + "items": "string" + } + }`; + + string[][] & readonly colors = [["red", "green", "blue"]]; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(colors); + string[][] & readonly deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -100,9 +165,9 @@ public isolated function testEnumArrays() returns error? { Numbers[] colors = ["ONE", "TWO", "THREE"]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - Numbers[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, colors); + byte[] encodedValue = check avro.toAvro(colors); + Numbers[] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -129,9 +194,9 @@ public isolated function testArrayOfEnumArrays() returns error? { Numbers[][] colors = [["ONE", "TWO", "THREE"], ["ONE", "TWO", "THREE"]]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - Numbers[][] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, colors); + byte[] encodedValue = check avro.toAvro(colors); + Numbers[][] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -149,9 +214,9 @@ public isolated function testFloatArrays() returns error? { float[] numbers = [22.4, 556.84350, 78.0327]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(numbers); - float[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, numbers); + byte[] encodedValue = check avro.toAvro(numbers); + float[] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, numbers); } @test:Config { @@ -169,9 +234,9 @@ public isolated function testDoubleArrays() returns error? { float[] numbers = [22.439475948, 556.843549485340, 78.032985693457]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(numbers); - float[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, numbers); + byte[] encodedValue = check avro.toAvro(numbers); + float[] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, numbers); } @test:Config { @@ -189,9 +254,9 @@ public isolated function testLongArrays() returns error? { int[] numbers = [223432, 55423326, 7823423]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(numbers); - int[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, numbers); + byte[] encodedValue = check avro.toAvro(numbers); + int[] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, numbers); } @test:Config { @@ -209,8 +274,8 @@ public isolated function testInvalidDecimalArrays() returns error? { decimal[] numbers = [22.439475948, 556.843549485340, 78.032985693457]; Schema avro = check new (schema); - byte[]|Error encode = avro.toAvro(numbers); - test:assertTrue(encode is Error); + byte[]|Error encodedValue = avro.toAvro(numbers); + test:assertTrue(encodedValue is Error); } @test:Config { @@ -228,9 +293,9 @@ public isolated function testBooleanArrays() returns error? { boolean[] numbers = [true, true, false]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(numbers); - boolean[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, numbers); + byte[] encodedValue = check avro.toAvro(numbers); + boolean[] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, numbers); } @test:Config { @@ -247,8 +312,8 @@ public isolated function testArraysWithAnydata() returns error? { anydata numbers = ["22.4".toBytes(), "556.84350", 78.0327]; Schema avro = check new (schema); - byte[]|Error encode = avro.toAvro(numbers); - test:assertTrue(encode is Error); + byte[]|Error encodedValue = avro.toAvro(numbers); + test:assertTrue(encodedValue is Error); } @test:Config { @@ -270,9 +335,9 @@ public isolated function testArraysWithFixed() returns error? { byte[][] numbers = ["22".toBytes(), "55".toBytes(), "78".toBytes()]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(numbers); - byte[][] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, numbers); + byte[] encodedValue = check avro.toAvro(numbers); + byte[][] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, numbers); } @test:Config { @@ -309,9 +374,48 @@ public isolated function testRecordsInArrays() returns error? { }]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(students); - Student[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, students); + byte[] encodedValue = check avro.toAvro(students); + Student[] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, students); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testRecordsInReadOnlyArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + }`; + + Student[] & readonly students = [{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }]; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(students); + Student[] & readonly deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, students); } @test:Config { @@ -357,7 +461,55 @@ public isolated function testRecordArraysInArrays() returns error? { }]]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(students); - Student[][] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, students); + byte[] encodedValue = check avro.toAvro(students); + Student[][] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, students); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testRecordArraysInReadOnlyArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "array", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + } + }`; + + Student[][] & readonly students = [[{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }], [{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }]]; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(students); + Student[][] & readonly deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, students); } diff --git a/ballerina/tests/byte_tests.bal b/ballerina/tests/byte_tests.bal index 724ff0d..af35157 100644 --- a/ballerina/tests/byte_tests.bal +++ b/ballerina/tests/byte_tests.bal @@ -15,6 +15,7 @@ // under the License. import ballerina/test; +import ballerina/io; @test:Config{ groups: ["record", "bytes"] @@ -37,8 +38,8 @@ public isolated function testRecordsWithBytes() returns error? { }; Schema avro = check new(schema); - byte[] encode = check avro.toAvro(student); - Student1 deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(student); + Student1 deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, student); } @@ -57,9 +58,9 @@ public isolated function testArraysWithBytes() returns error? { byte[][] numbers = ["22.4".toBytes(), "556.84350".toBytes(), "78.0327".toBytes()]; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(numbers); - byte[][] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, numbers); + byte[] encodedValue = check avro.toAvro(numbers); + byte[][] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, numbers); } @test:Config { @@ -76,46 +77,17 @@ public isolated function testBytes() returns error? { byte[] value = "5".toBytes(); Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - byte[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + byte[] encodedValue = check avro.toAvro(value); + byte[] deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, value); } @test:Config { groups: ["record", "map", "bytes"] } public isolated function testNestedRecordsWithBytes() returns error? { - string schema = string ` - { - "type": "record", - "name": "Lecturer4", - "fields": [ - { - "name": "name", - "type": { - "type": "map", - "values": "int" - } - }, - { - "name": "byteData", - "type": "bytes" - }, - { - "name": "instructor", - "type": { - "type": "record", - "name": "ByteRecord", - "fields": [ - { - "name": "byteData", - "type": "bytes" - } - ] - } - } - ] - }`; + string jsonFileName = string `tests/resources/schema_bytes.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Lecturer4 lecturer4 = { name: { diff --git a/ballerina/tests/map_tests.bal b/ballerina/tests/map_tests.bal index d00211c..6b314aa 100644 --- a/ballerina/tests/map_tests.bal +++ b/ballerina/tests/map_tests.bal @@ -15,6 +15,7 @@ // under the License. import ballerina/test; +import ballerina/io; @test:Config { groups: ["map", "bytes"] @@ -29,8 +30,8 @@ public isolated function testMapsWithBytes() returns error? { map colors = {"red": "0".toBytes(), "green": "1".toBytes(), "blue": "2".toBytes()}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -51,8 +52,8 @@ public isolated function testMapsWithFixed() returns error? { map colors = {"red": "0".toBytes(), "green": "1".toBytes(), "blue": "2".toBytes()}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -81,49 +82,93 @@ public isolated function testMapsOfFixedMaps() returns error? { "blue": {"r": "0".toBytes(), "g": "1".toBytes(), "b": "2".toBytes()} }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map> deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map> deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @test:Config { - groups: ["map", "record"] + groups: ["map", "record", "kkk"] } -public isolated function testMapsWithRecords() returns error? { - string schema = string ` - { - "type": "map", - "values": { - "type": "record", - "name": "Instructor", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "student", - "type": ["null", { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "subject", - "type": "string" - } - ] - }] - } - ] - }, - "default": {} - }`; +public isolated function testReadOnlyMapsWithReadOnlyRecords() returns error? { + string jsonFileName = string `tests/resources/schema_map.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + map & readonly instructors = { + "john": {name: "John", student: {name: "Alice", subject: "Math"}}, + "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, + "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} + }; + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(instructors); + map & readonly deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(instructors, deserialize); +} + +@test:Config { + groups: ["map", "record", "kkk"] +} +public isolated function testMapsWithRecordsWithReadOnly() returns error? { + string jsonFileName = string `tests/resources/schema_map_2.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + map & readonly instructors = { + "john": {name: "John", student: {name: "Alice", subject: "Math"}}, + "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, + "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} + }; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(instructors); + map & readonly deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(instructors, deserialize); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithReadOnlyRecordsWithReadOnly() returns error? { + string jsonFileName = string `tests/resources/schema_map_3.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + map & readonly instructors = { + "john": {name: "John", student: {name: "Alice", subject: "Math"}}, + "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, + "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} + }; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(instructors); + map & readonly deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(instructors, deserialize); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithReadOnlyRecords() returns error? { + string jsonFileName = string `tests/resources/schema_map_4.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + map instructors = { + "john": {name: "John", student: {name: "Alice", subject: "Math"}}, + "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, + "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} + }; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(instructors); + map deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(instructors, deserialize); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithRecords() returns error? { + string jsonFileName = string `tests/resources/schema_map_5.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); map instructors = { "john": {name: "John", student: {name: "Alice", subject: "Math"}}, @@ -132,8 +177,8 @@ public isolated function testMapsWithRecords() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(instructors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(instructors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(instructors, deserialize); } @@ -151,8 +196,8 @@ public isolated function testMapsWithInt() returns error? { map colors = {"red": 0, "green": 1, "blue": 2}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -174,8 +219,8 @@ public isolated function testMapsWithEnum() returns error? { map colors = {"red": "ONE", "green": "TWO", "blue": "THREE"}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -202,8 +247,8 @@ public isolated function testMapsWithEnumArrays() returns error? { map colors = {"red": ["ONE", "TWO", "THREE"], "green": ["ONE", "TWO", "THREE"], "blue": ["ONE", "TWO", "THREE"]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -221,8 +266,8 @@ public isolated function testMapsWithFloat() returns error? { map colors = {"red": 2.3453, "green": 435.563, "blue": 20347.23}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -240,8 +285,8 @@ public isolated function testMapsWithDouble() returns error? { map colors = {"red": 2.3453, "green": 435.563, "blue": 20347.23}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -264,8 +309,8 @@ public isolated function testMapsWithDoubleArray() returns error? { map colors = {"red": [2.3434253, 435.56433, 20347.22343], "green": [2.3452343, 435.56343, 20347.2423], "blue": [2.3453243, 435.56243, 20347.22343]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -283,8 +328,8 @@ public isolated function testMapsWithLong() returns error? { map colors = {"red": 2, "green": 435, "blue": 2034723}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -302,8 +347,8 @@ public isolated function testMapsWithStrings() returns error? { map colors = {"red": "2", "green": "435", "blue": "2034723"}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -321,8 +366,8 @@ public isolated function testMapsWithUnionTypes() returns error? { map colors = {"red": "2", "green": "435", "blue": "2034723"}; Schema avro = check new (schema); - byte[]|Error encode = avro.toAvro(colors); - test:assertTrue(encode is Error); + byte[]|Error encodedValue = avro.toAvro(colors); + test:assertTrue(encodedValue is Error); } @test:Config { @@ -339,8 +384,27 @@ public isolated function testMapsWithBoolean() returns error? { map colors = {"red": true, "green": false, "blue": false}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["map", "boolean", "ssq"] +} +public isolated function testMapsWithBooleanWithReadOnlyValues() returns error? { + string schema = string ` + { + "type": "map", + "values": "boolean", + "default": {} + }`; + + map & readonly colors = {"red": true, "green": false, "blue": false}; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(colors); + map & readonly deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -365,8 +429,37 @@ public isolated function testMapsWithMaps() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map> deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map> deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(colors, deserialize); +} + +@test:Config { + groups: ["map", "k"] +} +public isolated function testMapsWithNestedMapsWithReadOnlyValues() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "map", + "values": { + "type": "map", + "values": "int" + } + }, + "default": {} + }`; + + map>> & readonly colors = { + "red": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}}, + "green": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}}, + "blue": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}} + }; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(colors); + map>> & readonly deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -394,8 +487,8 @@ public isolated function testMapsWithNestedMaps() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map>> deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map>> deserialize = check avro.fromAvro(encodedValue); test:assertEquals(colors, deserialize); } @@ -416,8 +509,8 @@ public isolated function testMapsWithLongArray() returns error? { map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -438,8 +531,30 @@ public isolated function testMapsWithIntArray() returns error? { map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(deserialize, colors); +} + +@test:Config { + groups: ["map", "int", "az"] +} +public isolated function testMapsWithIntArrayWithReadOnlyValues() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "int" + }, + "default": {} + }`; + + map & readonly colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(colors); + map & readonly deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -460,8 +575,8 @@ public isolated function testMapsWithFloatArray() returns error? { map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -482,8 +597,8 @@ public isolated function testMapsWithStringArray() returns error? { map colors = {"red": ["252", "122", "41"], "green": ["235", "163", "23"], "blue": ["207", "123"]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -504,8 +619,8 @@ public isolated function testMapsWithUnionArray() returns error? { map colors = {"red": ["252", "122", "41"], "green": ["235", "163", "23"], "blue": ["207", "123"]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -526,8 +641,8 @@ public isolated function testMapsWithUnionIntArray() returns error? { map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -548,8 +663,8 @@ public isolated function testMapsWithUnionLongArray() returns error? { map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -570,8 +685,8 @@ public isolated function testMapsWithUnionFloatArray() returns error? { map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -592,8 +707,8 @@ public isolated function testMapsWithUnionDoubleArray() returns error? { map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -614,8 +729,8 @@ public isolated function testMapsWithBytesArray() returns error? { map colors = {"red": ["252".toBytes(), "122".toBytes(), "41".toBytes()], "green": ["235".toBytes(), "163".toBytes(), "23".toBytes()], "blue": ["207".toBytes(), "123".toBytes()]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -640,8 +755,8 @@ public isolated function testMapsWithFixedArray() returns error? { map colors = {"red": ["252".toBytes(), "122".toBytes(), "411".toBytes()], "green": ["235".toBytes(), "163".toBytes(), "213".toBytes()], "blue": ["207".toBytes(), "123".toBytes()]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } @@ -662,15 +777,70 @@ public isolated function testMapsWithBooleanArray() returns error? { map colors = {"red": [true, false, true], "green": [false, true, false], "blue": [true, false]}; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(colors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(deserialize, colors); } +// @test:Config { +// groups: ["map", "record", "ggg"] +// } +// public isolated function testMapsWithRecordArray() returns error? { +// string schema = string ` +// { +// "type": "map", +// "values": { +// "type": "array", +// "items": { +// "type": "array", +// "items": { +// "type": "record", +// "name": "Instructor", +// "fields": [ +// { +// "name": "name", +// "type": ["null", "string"] +// }, +// { +// "name": "student", +// "type": ["null", { +// "type": "record", +// "name": "Student", +// "fields": [ +// { +// "name": "name", +// "type": "string" +// }, +// { +// "name": "subject", +// "type": "string" +// } +// ] +// }] +// } +// ] +// } +// } +// }, +// "default": {} +// }`; + +// map instructors = { +// "john": [{name: "John", student: {name: "Alice", subject: "Math"}}, {name: "John", student: {name: "Alice", subject: "Math"}}], +// "doe": [{name: "Doe", student: {name: "Bob", subject: "Science"}}, {name: "Doe", student: {name: "Bob", subject: "Science"}}], +// "jane": [{name: "Jane", student: {name: "Charlie", subject: "English"}}, {name: "Jane", student: {name: "Charlie", subject: "English"}}] +// }; + +// Schema avro = check new (schema); +// byte[] encodedValue = check avro.toAvro(instructors); +// map deserialize = check avro.fromAvro(encodedValue); +// test:assertEquals(instructors, deserialize); +// } + @test:Config { groups: ["map", "record", "gg"] } -public isolated function testMapsWithRecordArray() returns error? { +public isolated function testMapsWithArrayOfRecordArray() returns error? { string schema = string ` { "type": "map", @@ -717,8 +887,8 @@ public isolated function testMapsWithRecordArray() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(instructors); - map deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(instructors); + map deserialize = check avro.fromAvro(encodedValue); test:assertEquals(instructors, deserialize); } @@ -749,9 +919,9 @@ public isolated function testArrayOfStringArrayMaps() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, colors); + byte[] encodedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -825,12 +995,11 @@ public isolated function testMapsWithNestedRecordMaps() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(lecturers); - map> deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(lecturers); + map> deserialize = check avro.fromAvro(encodedValue); test:assertEquals(lecturers, deserialize); } - @test:Config { groups: ["map", "record"] } @@ -897,7 +1066,151 @@ public isolated function testMapsWithNestedRecordArrayMaps() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(lecturers); - map> deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(lecturers); + map> deserialize = check avro.fromAvro(encodedValue); test:assertEquals(lecturers, deserialize); } + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithNestedRecordArrayReadOnlyMaps() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "map", + "values": { + "type": "array", + "items": { + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "instructor", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "subject", + "type": ["null", "string"] + } + ] + }] + } + ] + }] + } + ] + } + } + } + }`; + + + Lecturer[] lecs = [{name: "John", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}, + {name: "Doe", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}]; + + map mapValue = {"r": lecs, "g": lecs, "b": lecs}; + map> & readonly lecturers = { + "john": mapValue.cloneReadOnly(), + "doe": mapValue.cloneReadOnly(), + "jane": mapValue.cloneReadOnly() + }; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(lecturers); + map> & readonly deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(lecturers, deserialize); +} + +// @test:Config { +// groups: ["map", "record"] +// } +// public isolated function testMapsWithReadOnlyRecordArrayReadOnlyMaps() returns error? { +// string schema = string ` +// { +// "type": "map", +// "values": { +// "type": "map", +// "values": { +// "type": "array", +// "items": { +// "type": "record", +// "name": "Lecturer", +// "fields": [ +// { +// "name": "name", +// "type": ["null", "string"] +// }, +// { +// "name": "instructor", +// "type": ["null", { +// "type": "record", +// "name": "Instructor", +// "fields": [ +// { +// "name": "name", +// "type": ["null", "string"] +// }, +// { +// "name": "student", +// "type": ["null", { +// "type": "record", +// "name": "Student", +// "fields": [ +// { +// "name": "name", +// "type": ["null", "string"] +// }, +// { +// "name": "subject", +// "type": ["null", "string"] +// } +// ] +// }] +// } +// ] +// }] +// } +// ] +// } +// } +// } +// }`; + + +// Lecturer[] & readonly lecs = [{name: "John", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}, +// {name: "Doe", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}]; + +// map mapValue = {"r": lecs, "g": lecs, "b": lecs}; +// map> lecturers = { +// "john": mapValue, +// "doe": mapValue, +// "jane": mapValue +// }; + +// Schema avro = check new (schema); +// byte[] encodedValue = check avro.toAvro(lecturers); +// map> deserialize = check avro.fromAvro(encodedValue); +// test:assertEquals(lecturers, deserialize); +// } diff --git a/ballerina/tests/primitive_tests.bal b/ballerina/tests/primitive_tests.bal index 3f7d43b..cc22356 100644 --- a/ballerina/tests/primitive_tests.bal +++ b/ballerina/tests/primitive_tests.bal @@ -30,9 +30,9 @@ public isolated function testIntValue() returns error? { int value = 5; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - int deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + byte[] encodedValue = check avro.toAvro(value); + int deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, value); } @test:Config { @@ -49,9 +49,9 @@ public isolated function testFloatValue() returns error? { float value = 5.5; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - float deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + byte[] encodedValue = check avro.toAvro(value); + float deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, value); } @test:Config { @@ -68,9 +68,9 @@ public isolated function testDoubleValue() returns error? { float value = 5.5595; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - float deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + byte[] encodedValue = check avro.toAvro(value); + float deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, value); } @test:Config { @@ -86,9 +86,9 @@ public isolated function testLongValue() returns error? { int value = 555950000000000000; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - int deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + byte[] encodedValue = check avro.toAvro(value); + int deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, value); } @test:Config { @@ -104,9 +104,9 @@ public isolated function testStringValue() returns error? { string value = "test"; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - string deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + byte[] encodedValue = check avro.toAvro(value); + string deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, value); } @test:Config { @@ -122,9 +122,9 @@ public isolated function testBoolean() returns error? { boolean value = true; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - boolean deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + byte[] encodedValue = check avro.toAvro(value); + boolean deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, value); } @test:Config { @@ -139,9 +139,9 @@ public isolated function testNullValues() returns error? { }`; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(()); - () deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, ()); + byte[] encodedValue = check avro.toAvro(()); + () deserializedValue = check avro.fromAvro(encodedValue); + test:assertEquals(deserializedValue, ()); } @test:Config { @@ -156,6 +156,6 @@ public isolated function testNullValuesWithNonNullData() returns error? { }`; Schema avro = check new (schema); - byte[]|error encode = avro.toAvro("string"); - test:assertTrue(encode is error); + byte[]|error encodedValue = avro.toAvro("string"); + test:assertTrue(encodedValue is error); } diff --git a/ballerina/tests/record_tests.bal b/ballerina/tests/record_tests.bal index e986434..1a79a7c 100644 --- a/ballerina/tests/record_tests.bal +++ b/ballerina/tests/record_tests.bal @@ -63,8 +63,8 @@ public isolated function testRecordsWithDifferentTypeOfFields() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(student); - Person deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(student); + Person deserialize = check avro.fromAvro(encodedValue); test:assertEquals(student, deserialize); } @@ -168,10 +168,31 @@ public isolated function testArraysInRecords() returns error? { test:assertEquals(colors, deserialize); } -type Color1 record { - string name; - byte[] colors; -}; +@test:Config { + groups: ["record", "array"] +} +public isolated function testArraysInReadOnlyRecords() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "colors", "type": {"type": "array", "items": "string"}} + ] + }`; + + Color & readonly colors = { + name: "Red", + colors: ["maroon", "dark red", "light red"] + }; + + Schema avro = check new (schema); + byte[] serialize = check avro.toAvro(colors); + Color & readonly deserialize = check avro.fromAvro(serialize); + test:assertEquals(colors, deserialize); +} @test:Config { groups: ["record", "errors"] @@ -345,8 +366,8 @@ public isolated function testRecordsWithIntFields() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(student); - Person deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(student); + Person deserialize = check avro.fromAvro(encodedValue); test:assertEquals(student, deserialize); } @@ -371,8 +392,8 @@ public isolated function testRecordsWithLongFields() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(student); - Person deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(student); + Person deserialize = check avro.fromAvro(encodedValue); test:assertEquals(student, deserialize); } @@ -397,8 +418,8 @@ public isolated function testRecordsWithFloatFields() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(student); - Students deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(student); + Students deserialize = check avro.fromAvro(encodedValue); test:assertEquals(student, deserialize); } @@ -423,8 +444,8 @@ public isolated function testRecordsWithDoubleFields() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(student); - Students deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(student); + Students deserialize = check avro.fromAvro(encodedValue); test:assertEquals(student, deserialize); } @@ -449,8 +470,8 @@ public isolated function testRecordsWithBooleanFields() returns error? { }; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(student); - StudentRec deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(student); + StudentRec deserialize = check avro.fromAvro(encodedValue); test:assertEquals(student, deserialize); } diff --git a/ballerina/tests/test.bal b/ballerina/tests/test.bal index 9494089..be9d01d 100644 --- a/ballerina/tests/test.bal +++ b/ballerina/tests/test.bal @@ -17,6 +17,213 @@ import ballerina/io; import ballerina/test; +public type UnionEnumRecord record { + string|Numbers? field1; +}; + +public type UnionFixedRecord record { + string|byte[]? field1; +}; + +public type UnionRec record { + string|UnionEnumRecord? field1; +}; + +public type ReadOnlyRec readonly & record { + string|UnionEnumRecord? & readonly field1; +}; + +@test:Config { + groups: ["enum", "union"] +} +public isolated function testUnionEnums() returns error? { + string schema = string ` + { + "type": "record", + "name": "ExampleRecord", + "fields": [ + { + "name": "field1", + "type": ["string", { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO"] + }] + } + ] + }`; + + UnionEnumRecord number = { + field1: ONE + }; + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(number); + UnionEnumRecord deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(number, deserialize); +} + +@test:Config { + groups: ["fixed", "union"] +} +public isolated function testUnionFixed() returns error? { + string schema = string ` + { + "type": "record", + "name": "ExampleRecord", + "fields": [ + { + "name": "field1", + "type": [ + "string", + { + "type": "fixed", + "name": "Fixed", + "size": 2 + } + ] + } + ] + }`; + + UnionFixedRecord number = { + field1: "ON".toBytes() + }; + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(number); + UnionFixedRecord deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(number, deserialize); +} + +@test:Config { + groups: ["fixed", "union"] +} +public isolated function testUnionFixeWithReadOnlyValues() returns error? { + string schema = string ` + { + "type": "record", + "name": "ExampleRecord", + "fields": [ + { + "name": "field1", + "type": [ + "string", + { + "type": "fixed", + "name": "Fixed", + "size": 2 + } + ] + } + ] + }`; + + UnionFixedRecord & readonly number = { + field1: "ON".toBytes().cloneReadOnly() + }; + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(number); + UnionFixedRecord & readonly deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(number, deserialize); +} + +@test:Config { + groups: ["fixed", "union"] +} +public isolated function testUnionsWithRecordsAndStrings() returns error? { + string schema = string ` + { + "type": "record", + "name": "UnionRec", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "record", + "name": "UnionEnumRecord", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO"] + } + ] + } + ] + } + ] + } + ] + }`; + + UnionRec number = { + field1: { + field1: "ONE" + } + }; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(number); + UnionRec deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(number, deserialize); +} + +@test:Config { + groups: ["fixed", "union"] +} +public isolated function testUnionsWithReadOnlyRecords() returns error? { + string schema = string ` + { + "type": "record", + "name": "UnionRec", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "record", + "name": "UnionEnumRecord", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO"] + } + ] + } + ] + } + ] + } + ] + }`; + + ReadOnlyRec number = { + field1: { + field1: "ONE".cloneReadOnly() + } + }; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(number); + ReadOnlyRec deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(number, deserialize); +} + @test:Config { groups: ["enum"] } @@ -32,8 +239,28 @@ public isolated function testEnums() returns error? { Numbers number = "ONE"; Schema avro = check new (schema); - byte[] encode = check avro.toAvro(number); - Numbers deserialize = check avro.fromAvro(encode); + byte[] encodedValue = check avro.toAvro(number); + Numbers deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(number, deserialize); +} + +@test:Config { + groups: ["enum"] +} +public isolated function testEnumsWithReadOnlyValues() returns error? { + string schema = string ` + { + "type" : "enum", + "name" : "Numbers", + "namespace": "data", + "symbols" : [ "ONE", "TWO", "THREE", "FOUR" ] + }`; + + Numbers & readonly number = "ONE"; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(number); + Numbers & readonly deserialize = check avro.fromAvro(encodedValue); test:assertEquals(number, deserialize); } @@ -52,14 +279,14 @@ public isolated function testEnumsWithString() returns error? { string number = "FIVE"; Schema avro = check new (schema); - byte[]|Error encode = avro.toAvro(number); - test:assertTrue(encode is Error); + byte[]|Error encodedValue = avro.toAvro(number); + test:assertTrue(encodedValue is Error); } @test:Config { groups: ["fixed"] } -public isolated function testFixed() returns error? { +public isolated function testFixedWithInvalidSize() returns error? { string schema = string ` { "type": "fixed", @@ -67,18 +294,17 @@ public isolated function testFixed() returns error? { "size": 16 }`; - byte[] value = "u00ffffffffffffx".toBytes(); + byte[] value = "u00".toBytes(); Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - byte[] deserialize = check avro.fromAvro(encode); - test:assertEquals(deserialize, value); + byte[]|Error encodedValue = avro.toAvro(value); + test:assertTrue(encodedValue is Error); } @test:Config { groups: ["fixed"] } -public isolated function testFixedWithInvalidSize() returns error? { +public isolated function testFixed() returns error? { string schema = string ` { "type": "fixed", @@ -86,11 +312,12 @@ public isolated function testFixedWithInvalidSize() returns error? { "size": 16 }`; - byte[] value = "u00".toBytes(); + byte[] value = "u00ffffffffffffx".toBytes(); Schema avro = check new (schema); - byte[]|Error encode = avro.toAvro(value); - test:assertTrue(encode is Error); + byte[] encodedValue = check avro.toAvro(value); + byte[] deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(deserialize, value); } @test:Config { diff --git a/ballerina/tests/types.bal b/ballerina/tests/types.bal index 9b3f2c4..15ce307 100644 --- a/ballerina/tests/types.bal +++ b/ballerina/tests/types.bal @@ -125,6 +125,11 @@ public type Color record { string[] colors; }; +type Color1 record { + string name; + byte[] colors; +}; + public type FixedRec record { byte[] fixed_field; string other_field; diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java index ec85357..64150a7 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java @@ -18,26 +18,35 @@ package io.ballerina.lib.avro.deserialize.visitor; +import io.ballerina.lib.avro.Utils; import io.ballerina.lib.avro.deserialize.ArrayDeserializer; import io.ballerina.lib.avro.deserialize.Deserializer; +import io.ballerina.runtime.api.TypeTags; import io.ballerina.runtime.api.creators.ValueCreator; import io.ballerina.runtime.api.types.ArrayType; import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.values.BArray; import org.apache.avro.generic.GenericData; public class DeserializeArrayVisitor extends DeserializeVisitor { public Object visit(ArrayDeserializer arrayDeserializer, GenericData.Array data) throws Exception { Object[] objects = new Object[data.size()]; - Type arrayType = ((ArrayType) arrayDeserializer.getType()).getElementType(); + boolean isReadOnly = arrayDeserializer.getType().getTag() == TypeTags.INTERSECTION_TAG; + Type elementType = ((ArrayType) Utils.getMutableType(arrayDeserializer.getType())).getElementType(); int index = 0; for (Object element : data) { GenericData.Array dataArray = (GenericData.Array) element; - Type arrType = arrayType instanceof ArrayType ? arrayType : arrayDeserializer.getType(); + Type arrType = elementType.getTag() == TypeTags.ARRAY_TAG ? elementType : arrayDeserializer.getType(); objects[index++] = visitNestedArray(new ArrayDeserializer(arrayDeserializer.getSchema().getElementType(), arrType), dataArray); } - return ValueCreator.createArrayValue(objects, (ArrayType) arrayDeserializer.getType()); + BArray arrayValue = ValueCreator + .createArrayValue(objects, (ArrayType) Utils.getMutableType(arrayDeserializer.getType())); + if (isReadOnly) { + arrayValue.freezeDirect(); + } + return arrayValue; } public Object visitNestedArray(ArrayDeserializer arrayDeserializer, diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java index a7ee89b..c7e201c 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -18,6 +18,7 @@ package io.ballerina.lib.avro.deserialize.visitor; +import io.ballerina.lib.avro.Utils; import io.ballerina.lib.avro.deserialize.ArrayDeserializer; import io.ballerina.lib.avro.deserialize.Deserializer; import io.ballerina.lib.avro.deserialize.EnumDeserializer; @@ -30,7 +31,6 @@ import io.ballerina.runtime.api.creators.ValueCreator; import io.ballerina.runtime.api.types.ArrayType; import io.ballerina.runtime.api.types.Field; -import io.ballerina.runtime.api.types.IntersectionType; import io.ballerina.runtime.api.types.MapType; import io.ballerina.runtime.api.types.RecordType; import io.ballerina.runtime.api.types.ReferenceType; @@ -89,8 +89,10 @@ public BMap visit(RecordDeserializer recordDeserializer, Generi switch (field.schema().getType()) { case MAP -> processMapField(avroRecord, field, fieldData); - case ARRAY -> - processArrayField(avroRecord, field, fieldData); + case ARRAY -> { + Type fieldType = ((RecordType) avroRecord.getType()).getFields().get(field.name()).getFieldType(); + processArrayField(avroRecord, field, fieldData, fieldType); + } case BYTES -> processBytesField(avroRecord, field, fieldData); case RECORD -> @@ -124,7 +126,8 @@ public BMap visit(MapDeserializer mapDeserializer, Map - processMapArray(avroRecord, schema, (MapType) type, key, (GenericData.Array) value); + processMapArray(avroRecord, schema, + (MapType) getMutableType(type), key, (GenericData.Array) value); case BYTES -> avroRecord.put(StringUtils.fromString(key.toString()), ValueCreator.createArrayValue(((ByteBuffer) value).array())); @@ -135,12 +138,14 @@ public BMap visit(MapDeserializer mapDeserializer, Map - processMapRecord(avroRecord, schema, (MapType) type, key, (GenericRecord) value); + processMapRecord(avroRecord, schema, (MapType) getMutableType(type), + key, (GenericRecord) value); case FLOAT -> avroRecord.put(StringUtils.fromString(key.toString()), Double.parseDouble(value.toString())); case MAP -> - processMaps(avroRecord, schema, (MapType) type, key, (Map) value); + processMaps(avroRecord, schema, (MapType) getMutableType(type), + key, (Map) value); default -> avroRecord.put(StringUtils.fromString(key.toString()), value); } @@ -150,26 +155,27 @@ public BMap visit(MapDeserializer mapDeserializer, Map array = (GenericData.Array) data; switch (schema.getElementType().getType()) { case STRING -> { - return visitStringArray(array); + return ValueUtils.convert(visitStringArray(array), type); } case INT -> { - return visitIntArray(array); + return ValueUtils.convert(visitIntArray(array), type); } case LONG -> { - return visitLongArray(array); + return ValueUtils.convert(visitLongArray(array), type); } case FLOAT, DOUBLE -> { - return visitDoubleArray(array); + return ValueUtils.convert(visitDoubleArray(array), type); } case BOOLEAN -> { - return visitBooleanArray(array); + return ValueUtils.convert(visitBooleanArray(array), type); } default -> { - return visitBytesArray(array, primitiveDeserializer.getType()); + return ValueUtils.convert(visitBytesArray(array, primitiveDeserializer.getType()), type); } } } else { @@ -224,14 +230,15 @@ private BArray visitUnionArray(GenericData.Array data, ArrayType type, S public BArray visit(RecordDeserializer recordDeserializer, GenericData.Array data) throws Exception { List recordList = new ArrayList<>(); - Type type = recordDeserializer.getType(); + boolean isReadOnly = recordDeserializer.getType().getTag() == TypeTags.INTERSECTION_TAG; + Type type = Utils.getMutableType(recordDeserializer.getType()); Schema schema = recordDeserializer.getSchema(); switch (type.getTag()) { case TypeTags.ARRAY_TAG -> { for (Object datum : data) { - Type fieldType = ((ArrayType) type).getElementType().getCachedReferredType(); + Type fieldType = ((ArrayType) type).getElementType(); RecordDeserializer recordDes = new RecordDeserializer(schema.getElementType(), fieldType); - recordList.add(recordDes.visit(this, (GenericRecord) datum)); + recordList.add(recordDes.visit(this, datum)); } } case TypeTags.TYPE_REFERENCED_TYPE_TAG -> { @@ -242,7 +249,12 @@ public BArray visit(RecordDeserializer recordDeserializer, GenericData.Array createAvroRecord(Type type) { @@ -260,7 +272,7 @@ private void processMaps(BMap avroRecord, Schema schema, private void processMapRecord(BMap avroRecord, Schema schema, MapType type, Object key, GenericRecord value) throws Exception { - Type fieldType = type.getConstrainedType().getCachedReferredType(); + Type fieldType = type.getConstrainedType(); RecordDeserializer recordDes = new RecordDeserializer(schema.getValueType(), fieldType); Object fieldValue = recordDes.visit(this, value); avroRecord.put(fromString(key.toString()), fieldValue); @@ -405,7 +417,7 @@ public static Type extractMapType(Type type) throws Exception { case TypeTags.MAP_TAG -> mapType = fieldType; case TypeTags.INTERSECTION_TAG -> { - Type referredType = getMutableType((IntersectionType) fieldType); + Type referredType = getMutableType(fieldType); if (referredType.getTag() == TypeTags.MAP_TAG) { mapType = referredType; } From 6666d9c0b5811a77d343d3527b9a8a1ada8a249e Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Thu, 9 May 2024 21:57:27 +0530 Subject: [PATCH 35/47] Apply review suggestions --- .../java/io/ballerina/lib/avro/Utils.java | 27 +++++++++---------- .../avro/deserialize/visitor/RecordUtils.java | 4 +-- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/Utils.java b/native/src/main/java/io/ballerina/lib/avro/Utils.java index fbd4039..5ceb7d3 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Utils.java +++ b/native/src/main/java/io/ballerina/lib/avro/Utils.java @@ -45,23 +45,22 @@ public static BError createError(String message, Throwable throwable) { } public static Type getMutableType(Type dataType) { - if (dataType.getTag() == TypeTags.INTERSECTION_TAG) { - IntersectionType intersectionType = (IntersectionType) dataType; - for (Type type : intersectionType.getConstituentTypes()) { - Type referredType = TypeUtils.getImpliedType(type); - if (referredType.getTag() == TypeTags.UNION_TAG) { - for (Type elementType : ((UnionType) referredType).getMemberTypes()) { - if (elementType.getTag() == TypeTags.MAP_TAG) { - return elementType; - } + if (dataType.getTag() != TypeTags.INTERSECTION_TAG) { + return dataType; + } + IntersectionType intersectionType = (IntersectionType) dataType; + for (Type type : intersectionType.getConstituentTypes()) { + Type referredType = TypeUtils.getImpliedType(type); + if (referredType.getTag() == TypeTags.UNION_TAG) { + for (Type elementType : ((UnionType) referredType).getMemberTypes()) { + if (elementType.getTag() != TypeTags.NULL_TAG) { + return elementType; } } - if (TypeUtils.getImpliedType(intersectionType.getEffectiveType()).getTag() == referredType.getTag()) { - return referredType; - } } - } else { - return dataType; + if (TypeUtils.getImpliedType(intersectionType.getEffectiveType()).getTag() == referredType.getTag()) { + return referredType; + } } throw new IllegalStateException("Unsupported intersection type found."); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java index c219efb..e440c74 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java @@ -48,8 +48,8 @@ public static void processMapField(BMap avroRecord, } public static void processArrayField(BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - ArrayDeserializer arrayDes = new ArrayDeserializer(field.schema(), avroRecord.getType()); + Schema.Field field, Object fieldData, Type type) throws Exception { + ArrayDeserializer arrayDes = new ArrayDeserializer(field.schema(), type); Object fieldValue = arrayDes.visit(new DeserializeVisitor(), (GenericData.Array) fieldData); avroRecord.put(fromString(field.name()), fieldValue); } From e84391661a076f72f6d8f9d54831153a6c9c7925 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Fri, 10 May 2024 11:26:32 +0530 Subject: [PATCH 36/47] Move large schemas to separate files --- ballerina/tests/array_tests.bal | 95 +----- ballerina/tests/map_tests.bal | 282 ++---------------- ballerina/tests/record_tests.bal | 2 +- .../schema_array_readonly_arrays.json | 22 ++ .../schema_array_readonly_records.json | 19 ++ .../resources/schema_array_record_arrays.json | 22 ++ .../tests/resources/schema_array_records.json | 19 ++ ballerina/tests/resources/schema_bytes.json | 30 ++ .../resources/schema_map_array_record.json | 33 ++ .../schema_map_array_record_arrays.json | 37 +++ .../tests/resources/schema_map_readonly.json | 31 ++ .../schema_map_readonly_records.json | 31 ++ .../schema_map_readonly_records_readonly.json | 31 ++ .../resources/schema_map_record_array.json | 49 +++ .../schema_map_record_array_readonly.json | 49 +++ .../resources/schema_map_record_maps.json | 46 +++ .../tests/resources/schema_map_records.json | 31 ++ .../schema_map_records_readonly.json | 31 ++ ...hema_2.json => schema_nested_records.json} | 0 .../{schema_1.json => schema_records.json} | 0 .../tests/resources/schema_union_enums.json | 14 + .../tests/resources/schema_union_fixed.json | 17 ++ .../resources/schema_union_fixed_strings.json | 17 ++ .../tests/resources/schema_union_records.json | 31 ++ .../schema_union_records_strings.json | 31 ++ ballerina/tests/test.bal | 132 +------- 26 files changed, 647 insertions(+), 455 deletions(-) create mode 100644 ballerina/tests/resources/schema_array_readonly_arrays.json create mode 100644 ballerina/tests/resources/schema_array_readonly_records.json create mode 100644 ballerina/tests/resources/schema_array_record_arrays.json create mode 100644 ballerina/tests/resources/schema_array_records.json create mode 100644 ballerina/tests/resources/schema_bytes.json create mode 100644 ballerina/tests/resources/schema_map_array_record.json create mode 100644 ballerina/tests/resources/schema_map_array_record_arrays.json create mode 100644 ballerina/tests/resources/schema_map_readonly.json create mode 100644 ballerina/tests/resources/schema_map_readonly_records.json create mode 100644 ballerina/tests/resources/schema_map_readonly_records_readonly.json create mode 100644 ballerina/tests/resources/schema_map_record_array.json create mode 100644 ballerina/tests/resources/schema_map_record_array_readonly.json create mode 100644 ballerina/tests/resources/schema_map_record_maps.json create mode 100644 ballerina/tests/resources/schema_map_records.json create mode 100644 ballerina/tests/resources/schema_map_records_readonly.json rename ballerina/tests/resources/{schema_2.json => schema_nested_records.json} (100%) rename ballerina/tests/resources/{schema_1.json => schema_records.json} (100%) create mode 100644 ballerina/tests/resources/schema_union_enums.json create mode 100644 ballerina/tests/resources/schema_union_fixed.json create mode 100644 ballerina/tests/resources/schema_union_fixed_strings.json create mode 100644 ballerina/tests/resources/schema_union_records.json create mode 100644 ballerina/tests/resources/schema_union_records_strings.json diff --git a/ballerina/tests/array_tests.bal b/ballerina/tests/array_tests.bal index 07a0a83..bd46f99 100644 --- a/ballerina/tests/array_tests.bal +++ b/ballerina/tests/array_tests.bal @@ -15,6 +15,7 @@ // under the License. import ballerina/test; +import ballerina/io; @test:Config { groups: ["array", "int"] @@ -344,26 +345,8 @@ public isolated function testArraysWithFixed() returns error? { groups: ["record", "array"] } public isolated function testRecordsInArrays() returns error? { - string schema = string ` - { - "type": "array", - "name" : "recordArray", - "namespace": "data", - "items": { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": ["string", "null"] - }, - { - "name": "subject", - "type": ["string", "null"] - } - ] - } - }`; + string jsonFileName = string `tests/resources/schema_array_records.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Student[] students = [{ name: "Liam", @@ -383,26 +366,8 @@ public isolated function testRecordsInArrays() returns error? { groups: ["record", "array"] } public isolated function testRecordsInReadOnlyArrays() returns error? { - string schema = string ` - { - "type": "array", - "name" : "recordArray", - "namespace": "data", - "items": { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": ["string", "null"] - }, - { - "name": "subject", - "type": ["string", "null"] - } - ] - } - }`; + string jsonFileName = string `tests/resources/schema_array_readonly_records.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Student[] & readonly students = [{ name: "Liam", @@ -422,29 +387,8 @@ public isolated function testRecordsInReadOnlyArrays() returns error? { groups: ["record", "array"] } public isolated function testRecordArraysInArrays() returns error? { - string schema = string ` - { - "type": "array", - "name" : "recordArray", - "namespace": "data", - "items": { - "type": "array", - "items": { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": ["string", "null"] - }, - { - "name": "subject", - "type": ["string", "null"] - } - ] - } - } - }`; + string jsonFileName = string `tests/resources/schema_array_record_arrays.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Student[][] students = [[{ name: "Liam", @@ -470,29 +414,8 @@ public isolated function testRecordArraysInArrays() returns error? { groups: ["record", "array"] } public isolated function testRecordArraysInReadOnlyArrays() returns error? { - string schema = string ` - { - "type": "array", - "name" : "recordArray", - "namespace": "data", - "items": { - "type": "array", - "items": { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": ["string", "null"] - }, - { - "name": "subject", - "type": ["string", "null"] - } - ] - } - } - }`; + string jsonFileName = string `tests/resources/schema_array_readonly_arrays.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Student[][] & readonly students = [[{ name: "Liam", diff --git a/ballerina/tests/map_tests.bal b/ballerina/tests/map_tests.bal index 6b314aa..141ecad 100644 --- a/ballerina/tests/map_tests.bal +++ b/ballerina/tests/map_tests.bal @@ -91,7 +91,7 @@ public isolated function testMapsOfFixedMaps() returns error? { groups: ["map", "record", "kkk"] } public isolated function testReadOnlyMapsWithReadOnlyRecords() returns error? { - string jsonFileName = string `tests/resources/schema_map.json`; + string jsonFileName = string `tests/resources/schema_map_readonly.json`; string schema = (check io:fileReadJson(jsonFileName)).toString(); map & readonly instructors = { @@ -110,7 +110,7 @@ public isolated function testReadOnlyMapsWithReadOnlyRecords() returns error? { groups: ["map", "record", "kkk"] } public isolated function testMapsWithRecordsWithReadOnly() returns error? { - string jsonFileName = string `tests/resources/schema_map_2.json`; + string jsonFileName = string `tests/resources/schema_map_records_readonly.json`; string schema = (check io:fileReadJson(jsonFileName)).toString(); map & readonly instructors = { @@ -129,7 +129,7 @@ public isolated function testMapsWithRecordsWithReadOnly() returns error? { groups: ["map", "record"] } public isolated function testMapsWithReadOnlyRecordsWithReadOnly() returns error? { - string jsonFileName = string `tests/resources/schema_map_3.json`; + string jsonFileName = string `tests/resources/schema_map_readonly_records_readonly.json`; string schema = (check io:fileReadJson(jsonFileName)).toString(); map & readonly instructors = { @@ -148,7 +148,7 @@ public isolated function testMapsWithReadOnlyRecordsWithReadOnly() returns error groups: ["map", "record"] } public isolated function testMapsWithReadOnlyRecords() returns error? { - string jsonFileName = string `tests/resources/schema_map_4.json`; + string jsonFileName = string `tests/resources/schema_map_readonly_records.json`; string schema = (check io:fileReadJson(jsonFileName)).toString(); map instructors = { @@ -167,7 +167,7 @@ public isolated function testMapsWithReadOnlyRecords() returns error? { groups: ["map", "record"] } public isolated function testMapsWithRecords() returns error? { - string jsonFileName = string `tests/resources/schema_map_5.json`; + string jsonFileName = string `tests/resources/schema_map_records.json`; string schema = (check io:fileReadJson(jsonFileName)).toString(); map instructors = { @@ -782,103 +782,12 @@ public isolated function testMapsWithBooleanArray() returns error? { test:assertEquals(deserialize, colors); } -// @test:Config { -// groups: ["map", "record", "ggg"] -// } -// public isolated function testMapsWithRecordArray() returns error? { -// string schema = string ` -// { -// "type": "map", -// "values": { -// "type": "array", -// "items": { -// "type": "array", -// "items": { -// "type": "record", -// "name": "Instructor", -// "fields": [ -// { -// "name": "name", -// "type": ["null", "string"] -// }, -// { -// "name": "student", -// "type": ["null", { -// "type": "record", -// "name": "Student", -// "fields": [ -// { -// "name": "name", -// "type": "string" -// }, -// { -// "name": "subject", -// "type": "string" -// } -// ] -// }] -// } -// ] -// } -// } -// }, -// "default": {} -// }`; - -// map instructors = { -// "john": [{name: "John", student: {name: "Alice", subject: "Math"}}, {name: "John", student: {name: "Alice", subject: "Math"}}], -// "doe": [{name: "Doe", student: {name: "Bob", subject: "Science"}}, {name: "Doe", student: {name: "Bob", subject: "Science"}}], -// "jane": [{name: "Jane", student: {name: "Charlie", subject: "English"}}, {name: "Jane", student: {name: "Charlie", subject: "English"}}] -// }; - -// Schema avro = check new (schema); -// byte[] encodedValue = check avro.toAvro(instructors); -// map deserialize = check avro.fromAvro(encodedValue); -// test:assertEquals(instructors, deserialize); -// } - @test:Config { - groups: ["map", "record", "gg"] + groups: ["map", "record"] } public isolated function testMapsWithArrayOfRecordArray() returns error? { - string schema = string ` - { - "type": "map", - "values": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "record", - "name": "Instructor", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "student", - "type": ["null", { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "subject", - "type": "string" - } - ] - }] - } - ] - } - } - }, - "default": {} - }`; + string jsonFileName = string `tests/resources/schema_map_array_record_arrays.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); map instructors = { "john": [[{name: "John", student: {name: "Alice", subject: "Math"}}, {name: "John", student: {name: "Alice", subject: "Math"}}]], @@ -928,54 +837,8 @@ public isolated function testArrayOfStringArrayMaps() returns error? { groups: ["map", "record"] } public isolated function testMapsWithNestedRecordMaps() returns error? { - string schema = string ` - { - "type": "map", - "values": { - "type": "map", - "values": { - "type": "record", - "name": "Lecturer", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "instructor", - "type": ["null", { - "type": "record", - "name": "Instructor", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "student", - "type": ["null", { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "subject", - "type": ["null", "string"] - } - ] - }] - } - ] - }] - } - ] - } - } - }`; - + string jsonFileName = string `tests/resources/schema_map_record_maps.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Lecturer lec = { name: "John", @@ -1004,57 +867,8 @@ public isolated function testMapsWithNestedRecordMaps() returns error? { groups: ["map", "record"] } public isolated function testMapsWithNestedRecordArrayMaps() returns error? { - string schema = string ` - { - "type": "map", - "values": { - "type": "map", - "values": { - "type": "array", - "items": { - "type": "record", - "name": "Lecturer", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "instructor", - "type": ["null", { - "type": "record", - "name": "Instructor", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "student", - "type": ["null", { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "subject", - "type": ["null", "string"] - } - ] - }] - } - ] - }] - } - ] - } - } - } - }`; - + string jsonFileName = string `tests/resources/schema_map_record_array.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Lecturer[] lecs = [{name: "John", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}, {name: "Doe", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}]; @@ -1074,58 +888,28 @@ public isolated function testMapsWithNestedRecordArrayMaps() returns error? { @test:Config { groups: ["map", "record"] } -public isolated function testMapsWithNestedRecordArrayReadOnlyMaps() returns error? { - string schema = string ` - { - "type": "map", - "values": { - "type": "map", - "values": { - "type": "array", - "items": { - "type": "record", - "name": "Lecturer", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "instructor", - "type": ["null", { - "type": "record", - "name": "Instructor", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "student", - "type": ["null", { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "subject", - "type": ["null", "string"] - } - ] - }] - } - ] - }] - } - ] - } - } - } - }`; +public isolated function testMapsWithRecordArray() returns error? { + string jsonFileName = string `tests/resources/schema_map_array_record.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + map instructors = { + "john": [{name: "John", student: {name: "Alice", subject: "Math"}}, {name: "John", student: {name: "Alice", subject: "Math"}}], + "doe": [{name: "Doe", student: {name: "Bob", subject: "Science"}}, {name: "Doe", student: {name: "Bob", subject: "Science"}}], + "jane": [{name: "Jane", student: {name: "Charlie", subject: "English"}}, {name: "Jane", student: {name: "Charlie", subject: "English"}}] + }; + + Schema avro = check new (schema); + byte[] encodedValue = check avro.toAvro(instructors); + map deserialize = check avro.fromAvro(encodedValue); + test:assertEquals(instructors, deserialize); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithNestedRecordArrayReadOnlyMaps() returns error? { + string jsonFileName = string `tests/resources/schema_map_record_array_readonly.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Lecturer[] lecs = [{name: "John", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}, {name: "Doe", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}]; diff --git a/ballerina/tests/record_tests.bal b/ballerina/tests/record_tests.bal index 1a79a7c..af7a66d 100644 --- a/ballerina/tests/record_tests.bal +++ b/ballerina/tests/record_tests.bal @@ -169,7 +169,7 @@ public isolated function testArraysInRecords() returns error? { } @test:Config { - groups: ["record", "array"] + groups: ["record", "array", "aaaa"] } public isolated function testArraysInReadOnlyRecords() returns error? { string schema = string ` diff --git a/ballerina/tests/resources/schema_array_readonly_arrays.json b/ballerina/tests/resources/schema_array_readonly_arrays.json new file mode 100644 index 0000000..dfb920b --- /dev/null +++ b/ballerina/tests/resources/schema_array_readonly_arrays.json @@ -0,0 +1,22 @@ +{ + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "array", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + } +} diff --git a/ballerina/tests/resources/schema_array_readonly_records.json b/ballerina/tests/resources/schema_array_readonly_records.json new file mode 100644 index 0000000..6412977 --- /dev/null +++ b/ballerina/tests/resources/schema_array_readonly_records.json @@ -0,0 +1,19 @@ +{ + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } +} diff --git a/ballerina/tests/resources/schema_array_record_arrays.json b/ballerina/tests/resources/schema_array_record_arrays.json new file mode 100644 index 0000000..dfb920b --- /dev/null +++ b/ballerina/tests/resources/schema_array_record_arrays.json @@ -0,0 +1,22 @@ +{ + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "array", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + } +} diff --git a/ballerina/tests/resources/schema_array_records.json b/ballerina/tests/resources/schema_array_records.json new file mode 100644 index 0000000..6412977 --- /dev/null +++ b/ballerina/tests/resources/schema_array_records.json @@ -0,0 +1,19 @@ +{ + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } +} diff --git a/ballerina/tests/resources/schema_bytes.json b/ballerina/tests/resources/schema_bytes.json new file mode 100644 index 0000000..8daf0fa --- /dev/null +++ b/ballerina/tests/resources/schema_bytes.json @@ -0,0 +1,30 @@ +{ + "type": "record", + "name": "Lecturer4", + "fields": [ + { + "name": "name", + "type": { + "type": "map", + "values": "int" + } + }, + { + "name": "byteData", + "type": "bytes" + }, + { + "name": "instructor", + "type": { + "type": "record", + "name": "ByteRecord", + "fields": [ + { + "name": "byteData", + "type": "bytes" + } + ] + } + } + ] +} diff --git a/ballerina/tests/resources/schema_map_array_record.json b/ballerina/tests/resources/schema_map_array_record.json new file mode 100644 index 0000000..8ecb83c --- /dev/null +++ b/ballerina/tests/resources/schema_map_array_record.json @@ -0,0 +1,33 @@ +{ + "type": "map", + "values": { + "type": "array", + "items": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + } + } +} diff --git a/ballerina/tests/resources/schema_map_array_record_arrays.json b/ballerina/tests/resources/schema_map_array_record_arrays.json new file mode 100644 index 0000000..313cee1 --- /dev/null +++ b/ballerina/tests/resources/schema_map_array_record_arrays.json @@ -0,0 +1,37 @@ +{ + "type": "map", + "values": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + } + } + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_map_readonly.json b/ballerina/tests/resources/schema_map_readonly.json new file mode 100644 index 0000000..d85ea0b --- /dev/null +++ b/ballerina/tests/resources/schema_map_readonly.json @@ -0,0 +1,31 @@ +{ + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_map_readonly_records.json b/ballerina/tests/resources/schema_map_readonly_records.json new file mode 100644 index 0000000..d85ea0b --- /dev/null +++ b/ballerina/tests/resources/schema_map_readonly_records.json @@ -0,0 +1,31 @@ +{ + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_map_readonly_records_readonly.json b/ballerina/tests/resources/schema_map_readonly_records_readonly.json new file mode 100644 index 0000000..d85ea0b --- /dev/null +++ b/ballerina/tests/resources/schema_map_readonly_records_readonly.json @@ -0,0 +1,31 @@ +{ + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_map_record_array.json b/ballerina/tests/resources/schema_map_record_array.json new file mode 100644 index 0000000..70de00a --- /dev/null +++ b/ballerina/tests/resources/schema_map_record_array.json @@ -0,0 +1,49 @@ +{ + "type": "map", + "values": { + "type": "map", + "values": { + "type": "array", + "items": { + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "instructor", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "subject", + "type": ["null", "string"] + } + ] + }] + } + ] + }] + } + ] + } + } + } +} diff --git a/ballerina/tests/resources/schema_map_record_array_readonly.json b/ballerina/tests/resources/schema_map_record_array_readonly.json new file mode 100644 index 0000000..70de00a --- /dev/null +++ b/ballerina/tests/resources/schema_map_record_array_readonly.json @@ -0,0 +1,49 @@ +{ + "type": "map", + "values": { + "type": "map", + "values": { + "type": "array", + "items": { + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "instructor", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "subject", + "type": ["null", "string"] + } + ] + }] + } + ] + }] + } + ] + } + } + } +} diff --git a/ballerina/tests/resources/schema_map_record_maps.json b/ballerina/tests/resources/schema_map_record_maps.json new file mode 100644 index 0000000..8d572f2 --- /dev/null +++ b/ballerina/tests/resources/schema_map_record_maps.json @@ -0,0 +1,46 @@ +{ + "type": "map", + "values": { + "type": "map", + "values": { + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "instructor", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "subject", + "type": ["null", "string"] + } + ] + }] + } + ] + }] + } + ] + } + } +} diff --git a/ballerina/tests/resources/schema_map_records.json b/ballerina/tests/resources/schema_map_records.json new file mode 100644 index 0000000..d85ea0b --- /dev/null +++ b/ballerina/tests/resources/schema_map_records.json @@ -0,0 +1,31 @@ +{ + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_map_records_readonly.json b/ballerina/tests/resources/schema_map_records_readonly.json new file mode 100644 index 0000000..d85ea0b --- /dev/null +++ b/ballerina/tests/resources/schema_map_records_readonly.json @@ -0,0 +1,31 @@ +{ + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_2.json b/ballerina/tests/resources/schema_nested_records.json similarity index 100% rename from ballerina/tests/resources/schema_2.json rename to ballerina/tests/resources/schema_nested_records.json diff --git a/ballerina/tests/resources/schema_1.json b/ballerina/tests/resources/schema_records.json similarity index 100% rename from ballerina/tests/resources/schema_1.json rename to ballerina/tests/resources/schema_records.json diff --git a/ballerina/tests/resources/schema_union_enums.json b/ballerina/tests/resources/schema_union_enums.json new file mode 100644 index 0000000..583d686 --- /dev/null +++ b/ballerina/tests/resources/schema_union_enums.json @@ -0,0 +1,14 @@ +{ + "type": "record", + "name": "ExampleRecord", + "fields": [ + { + "name": "field1", + "type": ["string", { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO"] + }] + } + ] +} diff --git a/ballerina/tests/resources/schema_union_fixed.json b/ballerina/tests/resources/schema_union_fixed.json new file mode 100644 index 0000000..70c86ad --- /dev/null +++ b/ballerina/tests/resources/schema_union_fixed.json @@ -0,0 +1,17 @@ +{ + "type": "record", + "name": "ExampleRecord", + "fields": [ + { + "name": "field1", + "type": [ + "string", + { + "type": "fixed", + "name": "Fixed", + "size": 2 + } + ] + } + ] +} diff --git a/ballerina/tests/resources/schema_union_fixed_strings.json b/ballerina/tests/resources/schema_union_fixed_strings.json new file mode 100644 index 0000000..70c86ad --- /dev/null +++ b/ballerina/tests/resources/schema_union_fixed_strings.json @@ -0,0 +1,17 @@ +{ + "type": "record", + "name": "ExampleRecord", + "fields": [ + { + "name": "field1", + "type": [ + "string", + { + "type": "fixed", + "name": "Fixed", + "size": 2 + } + ] + } + ] +} diff --git a/ballerina/tests/resources/schema_union_records.json b/ballerina/tests/resources/schema_union_records.json new file mode 100644 index 0000000..e9d6f9a --- /dev/null +++ b/ballerina/tests/resources/schema_union_records.json @@ -0,0 +1,31 @@ +{ + "type": "record", + "name": "UnionRec", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "record", + "name": "UnionEnumRecord", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO"] + } + ] + } + ] + } + ] + } + ] +} diff --git a/ballerina/tests/resources/schema_union_records_strings.json b/ballerina/tests/resources/schema_union_records_strings.json new file mode 100644 index 0000000..e9d6f9a --- /dev/null +++ b/ballerina/tests/resources/schema_union_records_strings.json @@ -0,0 +1,31 @@ +{ + "type": "record", + "name": "UnionRec", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "record", + "name": "UnionEnumRecord", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO"] + } + ] + } + ] + } + ] + } + ] +} diff --git a/ballerina/tests/test.bal b/ballerina/tests/test.bal index be9d01d..6977eea 100644 --- a/ballerina/tests/test.bal +++ b/ballerina/tests/test.bal @@ -37,21 +37,8 @@ public type ReadOnlyRec readonly & record { groups: ["enum", "union"] } public isolated function testUnionEnums() returns error? { - string schema = string ` - { - "type": "record", - "name": "ExampleRecord", - "fields": [ - { - "name": "field1", - "type": ["string", { - "type": "enum", - "name": "Numbers", - "symbols": ["ONE", "TWO"] - }] - } - ] - }`; + string jsonFileName = string `tests/resources/schema_union_enums.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); UnionEnumRecord number = { field1: ONE @@ -66,24 +53,8 @@ public isolated function testUnionEnums() returns error? { groups: ["fixed", "union"] } public isolated function testUnionFixed() returns error? { - string schema = string ` - { - "type": "record", - "name": "ExampleRecord", - "fields": [ - { - "name": "field1", - "type": [ - "string", - { - "type": "fixed", - "name": "Fixed", - "size": 2 - } - ] - } - ] - }`; + string jsonFileName = string `tests/resources/schema_union_fixed.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); UnionFixedRecord number = { field1: "ON".toBytes() @@ -98,24 +69,8 @@ public isolated function testUnionFixed() returns error? { groups: ["fixed", "union"] } public isolated function testUnionFixeWithReadOnlyValues() returns error? { - string schema = string ` - { - "type": "record", - "name": "ExampleRecord", - "fields": [ - { - "name": "field1", - "type": [ - "string", - { - "type": "fixed", - "name": "Fixed", - "size": 2 - } - ] - } - ] - }`; + string jsonFileName = string `tests/resources/schema_union_fixed_strings.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); UnionFixedRecord & readonly number = { field1: "ON".toBytes().cloneReadOnly() @@ -130,38 +85,8 @@ public isolated function testUnionFixeWithReadOnlyValues() returns error? { groups: ["fixed", "union"] } public isolated function testUnionsWithRecordsAndStrings() returns error? { - string schema = string ` - { - "type": "record", - "name": "UnionRec", - "fields": [ - { - "name": "field1", - "type": [ - "null", - "string", - { - "type": "record", - "name": "UnionEnumRecord", - "fields": [ - { - "name": "field1", - "type": [ - "null", - "string", - { - "type": "enum", - "name": "Numbers", - "symbols": ["ONE", "TWO"] - } - ] - } - ] - } - ] - } - ] - }`; + string jsonFileName = string `tests/resources/schema_union_records_strings.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); UnionRec number = { field1: { @@ -179,38 +104,8 @@ public isolated function testUnionsWithRecordsAndStrings() returns error? { groups: ["fixed", "union"] } public isolated function testUnionsWithReadOnlyRecords() returns error? { - string schema = string ` - { - "type": "record", - "name": "UnionRec", - "fields": [ - { - "name": "field1", - "type": [ - "null", - "string", - { - "type": "record", - "name": "UnionEnumRecord", - "fields": [ - { - "name": "field1", - "type": [ - "null", - "string", - { - "type": "enum", - "name": "Numbers", - "symbols": ["ONE", "TWO"] - } - ] - } - ] - } - ] - } - ] - }`; + string jsonFileName = string `tests/resources/schema_union_records.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); ReadOnlyRec number = { field1: { @@ -354,9 +249,8 @@ public function testDbSchemaWithRecords() returns error? { groups: ["record"] } public function testComplexDbSchema() returns error? { - string jsonFileName = string `tests/resources/schema_1.json`; - json result = check io:fileReadJson(jsonFileName); - string schema = result.toString(); + string jsonFileName = string `tests/resources/schema_records.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Envelope envelope = { before: { @@ -430,7 +324,7 @@ public function testComplexDbSchema() returns error? { groups: ["record"] } public function testComplexDbSchemaWithNestedRecords() returns error? { - string jsonFileName = string `tests/resources/schema_2.json`; + string jsonFileName = string `tests/resources/schema_nested_records.json`; json result = check io:fileReadJson(jsonFileName); string schema = result.toString(); From 0db30cb35f6fa4dcf9e94c5d2e8235f9362053b4 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Fri, 10 May 2024 11:26:42 +0530 Subject: [PATCH 37/47] Fix review comments --- build-config/spotbugs-exclude.xml | 2 ++ native/src/main/java/io/ballerina/lib/avro/Utils.java | 8 -------- .../io/ballerina/lib/avro/deserialize/Deserializer.java | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/build-config/spotbugs-exclude.xml b/build-config/spotbugs-exclude.xml index 2079eda..687dc74 100644 --- a/build-config/spotbugs-exclude.xml +++ b/build-config/spotbugs-exclude.xml @@ -18,10 +18,12 @@ + + diff --git a/native/src/main/java/io/ballerina/lib/avro/Utils.java b/native/src/main/java/io/ballerina/lib/avro/Utils.java index 5ceb7d3..3f2bd5a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Utils.java +++ b/native/src/main/java/io/ballerina/lib/avro/Utils.java @@ -22,7 +22,6 @@ import io.ballerina.runtime.api.creators.ErrorCreator; import io.ballerina.runtime.api.types.IntersectionType; import io.ballerina.runtime.api.types.Type; -import io.ballerina.runtime.api.types.UnionType; import io.ballerina.runtime.api.utils.StringUtils; import io.ballerina.runtime.api.utils.TypeUtils; import io.ballerina.runtime.api.values.BError; @@ -51,13 +50,6 @@ public static Type getMutableType(Type dataType) { IntersectionType intersectionType = (IntersectionType) dataType; for (Type type : intersectionType.getConstituentTypes()) { Type referredType = TypeUtils.getImpliedType(type); - if (referredType.getTag() == TypeTags.UNION_TAG) { - for (Type elementType : ((UnionType) referredType).getMemberTypes()) { - if (elementType.getTag() != TypeTags.NULL_TAG) { - return elementType; - } - } - } if (TypeUtils.getImpliedType(intersectionType.getEffectiveType()).getTag() == referredType.getTag()) { return referredType; } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java index 744bb4a..ab023c7 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java @@ -30,7 +30,7 @@ public abstract class Deserializer { private final Type type; public Deserializer() { - this(null); + this(null, null); } public Deserializer(Type type) { From dc4d1b64326975c9a37fc97c8562c6a509064114 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Fri, 10 May 2024 14:00:20 +0530 Subject: [PATCH 38/47] Exclude spotbugs using the method names --- build-config/spotbugs-exclude.xml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build-config/spotbugs-exclude.xml b/build-config/spotbugs-exclude.xml index 687dc74..2895ed4 100644 --- a/build-config/spotbugs-exclude.xml +++ b/build-config/spotbugs-exclude.xml @@ -18,12 +18,15 @@ - + - + + + + From 2e97eb4610c82664b338f453c3f2d46ab7d61755 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Fri, 10 May 2024 14:07:57 +0530 Subject: [PATCH 39/47] Fix a review comment --- .../lib/avro/deserialize/visitor/DeserializeVisitor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java index c7e201c..1d891bf 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -445,9 +445,8 @@ public static RecordType extractRecordType(RecordType type) { case TypeTags.RECORD_TYPE_TAG -> recType = (RecordType) fieldType; case TypeTags.INTERSECTION_TAG -> { - Type getType = getMutableType(fieldType); - if (getType.getTag() == TypeTags.RECORD_TYPE_TAG) { - recType = (RecordType) getType; + if (getMutableType(fieldType).getTag() == TypeTags.RECORD_TYPE_TAG) { + recType = (RecordType) getMutableType(fieldType); } } default -> { From 4702208390ce12495ad2f67bf9f68fb06a03d23e Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Fri, 10 May 2024 14:33:40 +0530 Subject: [PATCH 40/47] Change the parameter order in Deserializer classes --- .../avro/deserialize/ArrayDeserializer.java | 2 +- .../avro/deserialize/DeserializeFactory.java | 8 +++---- .../avro/deserialize/FixedDeserializer.java | 2 +- .../deserialize/PrimitiveDeserializer.java | 2 +- .../avro/deserialize/RecordDeserializer.java | 2 +- .../avro/deserialize/UnionDeserializer.java | 2 +- .../visitor/DeserializeArrayVisitor.java | 5 ++-- .../visitor/DeserializeRecordVisitor.java | 4 ++-- .../visitor/DeserializeVisitor.java | 24 +++++++++---------- .../avro/deserialize/visitor/RecordUtils.java | 4 ++-- .../deserialize/visitor/UnionRecordUtils.java | 2 +- 11 files changed, 29 insertions(+), 28 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java index 5a1872a..5fc0c18 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java @@ -25,7 +25,7 @@ public class ArrayDeserializer extends Deserializer { - public ArrayDeserializer(Schema schema, Type type) { + public ArrayDeserializer(Type type, Schema schema) { super(type, schema); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java index 85e7438..3be1944 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java @@ -28,12 +28,12 @@ public static Deserializer generateDeserializer(Schema schema, Type type) { case NULL -> new NullDeserializer(); case FLOAT, DOUBLE -> new DoubleDeserializer(); case STRING, ENUM -> new StringDeserializer(); - case ARRAY -> new ArrayDeserializer(schema, type); - case FIXED -> new FixedDeserializer(schema, type); + case ARRAY -> new ArrayDeserializer(type, schema); + case FIXED -> new FixedDeserializer(type, schema); case MAP -> new MapDeserializer(schema, type); - case RECORD -> new RecordDeserializer(schema, type); + case RECORD -> new RecordDeserializer(type, schema); case BYTES -> new ByteDeserializer(); - default -> new PrimitiveDeserializer(schema, type); + default -> new PrimitiveDeserializer(type, schema); }; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java index cf22612..f823d4b 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java @@ -25,7 +25,7 @@ public class FixedDeserializer extends Deserializer { - public FixedDeserializer(Schema schema, Type type) { + public FixedDeserializer(Type type, Schema schema) { super(type, schema); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java index 555456f..a865b45 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java @@ -25,7 +25,7 @@ public class PrimitiveDeserializer extends Deserializer { - public PrimitiveDeserializer(Schema schema, Type type) { + public PrimitiveDeserializer(Type type, Schema schema) { super(type, schema); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java index aa48398..e27daa1 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java @@ -26,7 +26,7 @@ public class RecordDeserializer extends Deserializer { - public RecordDeserializer(Schema schema, Type type) { + public RecordDeserializer(Type type, Schema schema) { super(type, schema); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java index 980536b..3a6ea6f 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java @@ -25,7 +25,7 @@ public class UnionDeserializer extends Deserializer { - public UnionDeserializer(Schema schema, Type type) { + public UnionDeserializer(Type type, Schema schema) { super(type, schema); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java index 64150a7..455802e 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java @@ -26,6 +26,7 @@ import io.ballerina.runtime.api.types.ArrayType; import io.ballerina.runtime.api.types.Type; import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; public class DeserializeArrayVisitor extends DeserializeVisitor { @@ -38,8 +39,8 @@ public Object visit(ArrayDeserializer arrayDeserializer, GenericData.Array dataArray = (GenericData.Array) element; Type arrType = elementType.getTag() == TypeTags.ARRAY_TAG ? elementType : arrayDeserializer.getType(); - objects[index++] = visitNestedArray(new ArrayDeserializer(arrayDeserializer.getSchema().getElementType(), - arrType), dataArray); + Schema elementSchema = arrayDeserializer.getSchema().getElementType(); + objects[index++] = visitNestedArray(new ArrayDeserializer(arrType, elementSchema), dataArray); } BArray arrayValue = ValueCreator .createArrayValue(objects, (ArrayType) Utils.getMutableType(arrayDeserializer.getType())); diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java index 5ff489f..be711e8 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java @@ -84,7 +84,7 @@ private void processMapField(BMap avroRecord, private void processArrayField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { - ArrayDeserializer arrayDes = new ArrayDeserializer(field.schema(), avroRecord.getType()); + ArrayDeserializer arrayDes = new ArrayDeserializer(avroRecord.getType(), field.schema()); Object fieldValue = arrayDes.visit(this, (GenericData.Array) fieldData); avroRecord.put(StringUtils.fromString(field.name()), fieldValue); } @@ -98,7 +98,7 @@ private void processBytesField(BMap avroRecord, Schema.Field fi private void processRecordField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { Type recType = extractRecordType((RecordType) avroRecord.getType()); - RecordDeserializer recordDes = new RecordDeserializer(field.schema(), recType); + RecordDeserializer recordDes = new RecordDeserializer(recType, field.schema()); Object fieldValue = recordDes.visit(this, (GenericRecord) fieldData); avroRecord.put(StringUtils.fromString(field.name()), fieldValue); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java index 1d891bf..419a857 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -65,17 +65,17 @@ public class DeserializeVisitor implements IDeserializeVisitor { public static Deserializer createDeserializer(Schema schema, Type type) { return switch (schema.getElementType().getType()) { case UNION -> - new UnionDeserializer(schema, type); + new UnionDeserializer(type, schema); case ARRAY -> - new ArrayDeserializer(schema, type); + new ArrayDeserializer(type, schema); case ENUM -> - new EnumDeserializer(type); + new EnumDeserializer(type, schema); case RECORD -> - new RecordDeserializer(schema, type); + new RecordDeserializer(type, schema); case FIXED -> - new FixedDeserializer(schema, type); + new FixedDeserializer(type, schema); default -> - new PrimitiveDeserializer(schema, type); + new PrimitiveDeserializer(type, schema); }; } @@ -212,14 +212,14 @@ public BArray visit(UnionDeserializer unionDeserializer, GenericData.Array data, Type type, Schema schema) throws Exception { - RecordDeserializer recordDeserializer = new RecordDeserializer(schema.getElementType(), type); + RecordDeserializer recordDeserializer = new RecordDeserializer(type, schema.getElementType()); return (BArray) recordDeserializer.visit(this, data); } private BArray visitUnionArray(GenericData.Array data, ArrayType type, Schema schema) throws Exception { Object[] objects = new Object[data.size()]; Type elementType = type.getElementType(); - ArrayDeserializer arrayDeserializer = new ArrayDeserializer(schema.getElementType(), elementType); + ArrayDeserializer arrayDeserializer = new ArrayDeserializer(elementType, schema.getElementType()); int index = 0; for (Object currentData : data) { Object deserializedObject = arrayDeserializer.visit(this, (GenericData.Array) currentData); @@ -237,14 +237,14 @@ public BArray visit(RecordDeserializer recordDeserializer, GenericData.Array { for (Object datum : data) { Type fieldType = ((ArrayType) type).getElementType(); - RecordDeserializer recordDes = new RecordDeserializer(schema.getElementType(), fieldType); + RecordDeserializer recordDes = new RecordDeserializer(fieldType, schema.getElementType()); recordList.add(recordDes.visit(this, datum)); } } case TypeTags.TYPE_REFERENCED_TYPE_TAG -> { for (Object datum : data) { Type fieldType = ((ReferenceType) type).getReferredType(); - RecordDeserializer recordDes = new RecordDeserializer(schema.getElementType(), fieldType); + RecordDeserializer recordDes = new RecordDeserializer(fieldType, schema.getElementType()); recordList.add(recordDes.visit(this, (GenericRecord) datum)); } } @@ -273,7 +273,7 @@ private void processMaps(BMap avroRecord, Schema schema, private void processMapRecord(BMap avroRecord, Schema schema, MapType type, Object key, GenericRecord value) throws Exception { Type fieldType = type.getConstrainedType(); - RecordDeserializer recordDes = new RecordDeserializer(schema.getValueType(), fieldType); + RecordDeserializer recordDes = new RecordDeserializer(fieldType, schema.getValueType()); Object fieldValue = recordDes.visit(this, value); avroRecord.put(fromString(key.toString()), fieldValue); } @@ -281,7 +281,7 @@ private void processMapRecord(BMap avroRecord, Schema schema, private void processMapArray(BMap avroRecord, Schema schema, MapType type, Object key, GenericData.Array value) throws Exception { Type fieldType = type.getConstrainedType(); - ArrayDeserializer arrayDeserializer = new ArrayDeserializer(schema.getValueType(), fieldType); + ArrayDeserializer arrayDeserializer = new ArrayDeserializer(fieldType, schema.getValueType()); Object fieldValue = visit(arrayDeserializer, value); avroRecord.put(fromString(key.toString()), fieldValue); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java index e440c74..2e031a8 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java @@ -49,7 +49,7 @@ public static void processMapField(BMap avroRecord, public static void processArrayField(BMap avroRecord, Schema.Field field, Object fieldData, Type type) throws Exception { - ArrayDeserializer arrayDes = new ArrayDeserializer(field.schema(), type); + ArrayDeserializer arrayDes = new ArrayDeserializer(type, field.schema()); Object fieldValue = arrayDes.visit(new DeserializeVisitor(), (GenericData.Array) fieldData); avroRecord.put(fromString(field.name()), fieldValue); } @@ -63,7 +63,7 @@ public static void processBytesField(BMap avroRecord, Schema.Fi public static void processRecordField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { Type recType = extractRecordType((RecordType) avroRecord.getType()); - RecordDeserializer recordDes = new RecordDeserializer(field.schema(), recType); + RecordDeserializer recordDes = new RecordDeserializer(recType, field.schema()); Object fieldValue = recordDes.visit(new DeserializeVisitor(), fieldData); avroRecord.put(fromString(field.name()), fieldValue); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java index 0f98feb..b1f1488 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java @@ -115,7 +115,7 @@ private static void handleStringField(Schema.Field field, Object fieldData, BMap public static void handleRecordField(Type type, Schema.Field field, Object fieldData, BMap ballerinaRecord, Schema schemaType) throws Exception { if (fieldData instanceof GenericRecord) { - RecordDeserializer recordDes = new RecordDeserializer(schemaType, type); + RecordDeserializer recordDes = new RecordDeserializer(type, schemaType); Object fieldValue = recordDes.visit(new DeserializeVisitor(), (GenericRecord) fieldData); ballerinaRecord.put(StringUtils.fromString(field.name()), fieldValue); } From cd8b2078be1113b8d593cf26c79b2a53a4b2f8a3 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Fri, 10 May 2024 14:33:58 +0530 Subject: [PATCH 41/47] Remove redundant constructor in Deserializer --- .../io/ballerina/lib/avro/deserialize/Deserializer.java | 6 +----- .../io/ballerina/lib/avro/deserialize/EnumDeserializer.java | 5 +++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java index ab023c7..d7897a7 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java @@ -32,11 +32,7 @@ public abstract class Deserializer { public Deserializer() { this(null, null); } - - public Deserializer(Type type) { - this(type, null); - } - + public Deserializer(Type type, Schema schema) { this.schema = schema == null ? null : new Schema.Parser().parse(schema.toString()); this.type = type == null ? null : TypeUtils.getReferredType(type); diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java index f4770de..45818fe 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java @@ -20,12 +20,13 @@ import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; public class EnumDeserializer extends Deserializer { - public EnumDeserializer(Type type) { - super(type); + public EnumDeserializer(Type type, Schema schema) { + super(type, schema); } @Override From 1423760aa9e578044fdfdd773aa6e9fdd86bee7f Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Fri, 10 May 2024 14:48:16 +0530 Subject: [PATCH 42/47] Update variable names in tests --- ballerina/tests/array_tests.bal | 76 ++++----- ballerina/tests/byte_tests.bal | 18 +- ballerina/tests/map_tests.bal | 248 ++++++++++++++-------------- ballerina/tests/primitive_tests.bal | 32 ++-- ballerina/tests/record_tests.bal | 72 ++++---- ballerina/tests/test.bal | 68 ++++---- 6 files changed, 257 insertions(+), 257 deletions(-) diff --git a/ballerina/tests/array_tests.bal b/ballerina/tests/array_tests.bal index bd46f99..509e71a 100644 --- a/ballerina/tests/array_tests.bal +++ b/ballerina/tests/array_tests.bal @@ -32,8 +32,8 @@ public isolated function testIntArrays() returns error? { int[] numbers = [22, 556, 78]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(numbers); - int[] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(numbers); + int[] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, numbers); } @@ -52,8 +52,8 @@ public isolated function testReadOnlyIntArrays() returns error? { int[] & readonly numbers = [22, 556, 78]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(numbers); - int[] & readonly deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(numbers); + int[] & readonly deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, numbers); } @@ -72,8 +72,8 @@ public isolated function testStringArrays() returns error? { string[] colors = ["red", "green", "blue"]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - string[] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(colors); + string[] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, colors); } @@ -92,8 +92,8 @@ public isolated function testReadOnlyStringArrays() returns error? { string[] & readonly colors = ["red", "green", "blue"]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - string[] & readonly deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(colors); + string[] & readonly deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, colors); } @@ -117,8 +117,8 @@ public isolated function testArrayOfStringArrays() returns error? { string[][] colors = [["red", "green", "blue"]]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - string[][] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(colors); + string[][] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, colors); } @@ -142,8 +142,8 @@ public isolated function testReadOnlyArrayOfStringArrays() returns error? { string[][] & readonly colors = [["red", "green", "blue"]]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - string[][] & readonly deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(colors); + string[][] & readonly deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, colors); } @@ -166,8 +166,8 @@ public isolated function testEnumArrays() returns error? { Numbers[] colors = ["ONE", "TWO", "THREE"]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - Numbers[] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(colors); + Numbers[] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, colors); } @@ -195,8 +195,8 @@ public isolated function testArrayOfEnumArrays() returns error? { Numbers[][] colors = [["ONE", "TWO", "THREE"], ["ONE", "TWO", "THREE"]]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - Numbers[][] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(colors); + Numbers[][] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, colors); } @@ -215,8 +215,8 @@ public isolated function testFloatArrays() returns error? { float[] numbers = [22.4, 556.84350, 78.0327]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(numbers); - float[] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(numbers); + float[] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, numbers); } @@ -235,8 +235,8 @@ public isolated function testDoubleArrays() returns error? { float[] numbers = [22.439475948, 556.843549485340, 78.032985693457]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(numbers); - float[] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(numbers); + float[] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, numbers); } @@ -255,8 +255,8 @@ public isolated function testLongArrays() returns error? { int[] numbers = [223432, 55423326, 7823423]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(numbers); - int[] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(numbers); + int[] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, numbers); } @@ -275,8 +275,8 @@ public isolated function testInvalidDecimalArrays() returns error? { decimal[] numbers = [22.439475948, 556.843549485340, 78.032985693457]; Schema avro = check new (schema); - byte[]|Error encodedValue = avro.toAvro(numbers); - test:assertTrue(encodedValue is Error); + byte[]|Error serializedValue = avro.toAvro(numbers); + test:assertTrue(serializedValue is Error); } @test:Config { @@ -294,8 +294,8 @@ public isolated function testBooleanArrays() returns error? { boolean[] numbers = [true, true, false]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(numbers); - boolean[] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(numbers); + boolean[] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, numbers); } @@ -313,8 +313,8 @@ public isolated function testArraysWithAnydata() returns error? { anydata numbers = ["22.4".toBytes(), "556.84350", 78.0327]; Schema avro = check new (schema); - byte[]|Error encodedValue = avro.toAvro(numbers); - test:assertTrue(encodedValue is Error); + byte[]|Error serializedValue = avro.toAvro(numbers); + test:assertTrue(serializedValue is Error); } @test:Config { @@ -336,8 +336,8 @@ public isolated function testArraysWithFixed() returns error? { byte[][] numbers = ["22".toBytes(), "55".toBytes(), "78".toBytes()]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(numbers); - byte[][] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(numbers); + byte[][] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, numbers); } @@ -357,8 +357,8 @@ public isolated function testRecordsInArrays() returns error? { }]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(students); - Student[] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(students); + Student[] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, students); } @@ -378,8 +378,8 @@ public isolated function testRecordsInReadOnlyArrays() returns error? { }]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(students); - Student[] & readonly deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(students); + Student[] & readonly deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, students); } @@ -405,8 +405,8 @@ public isolated function testRecordArraysInArrays() returns error? { }]]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(students); - Student[][] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(students); + Student[][] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, students); } @@ -432,7 +432,7 @@ public isolated function testRecordArraysInReadOnlyArrays() returns error? { }]]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(students); - Student[][] & readonly deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(students); + Student[][] & readonly deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, students); } diff --git a/ballerina/tests/byte_tests.bal b/ballerina/tests/byte_tests.bal index af35157..b7f40a3 100644 --- a/ballerina/tests/byte_tests.bal +++ b/ballerina/tests/byte_tests.bal @@ -38,9 +38,9 @@ public isolated function testRecordsWithBytes() returns error? { }; Schema avro = check new(schema); - byte[] encodedValue = check avro.toAvro(student); - Student1 deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, student); + byte[] serializedValue = check avro.toAvro(student); + Student1 deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, student); } @test:Config { @@ -58,8 +58,8 @@ public isolated function testArraysWithBytes() returns error? { byte[][] numbers = ["22.4".toBytes(), "556.84350".toBytes(), "78.0327".toBytes()]; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(numbers); - byte[][] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(numbers); + byte[][] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, numbers); } @@ -77,8 +77,8 @@ public isolated function testBytes() returns error? { byte[] value = "5".toBytes(); Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(value); - byte[] deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(value); + byte[] deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, value); } @@ -103,7 +103,7 @@ public isolated function testNestedRecordsWithBytes() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(lecturer4); - Lecturer4 deserialize = check avro.fromAvro(serialize); + Lecturer4 deserializedValue = check avro.fromAvro(serialize); // deserialize.instructor.student.name = "Sam"; - test:assertEquals(deserialize, lecturer4); + test:assertEquals(deserializedValue, lecturer4); } diff --git a/ballerina/tests/map_tests.bal b/ballerina/tests/map_tests.bal index 141ecad..f756a11 100644 --- a/ballerina/tests/map_tests.bal +++ b/ballerina/tests/map_tests.bal @@ -30,9 +30,9 @@ public isolated function testMapsWithBytes() returns error? { map colors = {"red": "0".toBytes(), "green": "1".toBytes(), "blue": "2".toBytes()}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -52,9 +52,9 @@ public isolated function testMapsWithFixed() returns error? { map colors = {"red": "0".toBytes(), "green": "1".toBytes(), "blue": "2".toBytes()}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -82,9 +82,9 @@ public isolated function testMapsOfFixedMaps() returns error? { "blue": {"r": "0".toBytes(), "g": "1".toBytes(), "b": "2".toBytes()} }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map> deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map> deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -101,9 +101,9 @@ public isolated function testReadOnlyMapsWithReadOnlyRecords() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(instructors); - map & readonly deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(instructors, deserialize); + byte[] serializedValue = check avro.toAvro(instructors); + map & readonly deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(instructors, deserializedValue); } @test:Config { @@ -120,9 +120,9 @@ public isolated function testMapsWithRecordsWithReadOnly() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(instructors); - map & readonly deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(instructors, deserialize); + byte[] serializedValue = check avro.toAvro(instructors); + map & readonly deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(instructors, deserializedValue); } @test:Config { @@ -139,9 +139,9 @@ public isolated function testMapsWithReadOnlyRecordsWithReadOnly() returns error }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(instructors); - map & readonly deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(instructors, deserialize); + byte[] serializedValue = check avro.toAvro(instructors); + map & readonly deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(instructors, deserializedValue); } @test:Config { @@ -158,9 +158,9 @@ public isolated function testMapsWithReadOnlyRecords() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(instructors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(instructors, deserialize); + byte[] serializedValue = check avro.toAvro(instructors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(instructors, deserializedValue); } @test:Config { @@ -177,9 +177,9 @@ public isolated function testMapsWithRecords() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(instructors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(instructors, deserialize); + byte[] serializedValue = check avro.toAvro(instructors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(instructors, deserializedValue); } @test:Config { @@ -196,9 +196,9 @@ public isolated function testMapsWithInt() returns error? { map colors = {"red": 0, "green": 1, "blue": 2}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -219,9 +219,9 @@ public isolated function testMapsWithEnum() returns error? { map colors = {"red": "ONE", "green": "TWO", "blue": "THREE"}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -247,9 +247,9 @@ public isolated function testMapsWithEnumArrays() returns error? { map colors = {"red": ["ONE", "TWO", "THREE"], "green": ["ONE", "TWO", "THREE"], "blue": ["ONE", "TWO", "THREE"]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -266,9 +266,9 @@ public isolated function testMapsWithFloat() returns error? { map colors = {"red": 2.3453, "green": 435.563, "blue": 20347.23}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -285,9 +285,9 @@ public isolated function testMapsWithDouble() returns error? { map colors = {"red": 2.3453, "green": 435.563, "blue": 20347.23}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -309,9 +309,9 @@ public isolated function testMapsWithDoubleArray() returns error? { map colors = {"red": [2.3434253, 435.56433, 20347.22343], "green": [2.3452343, 435.56343, 20347.2423], "blue": [2.3453243, 435.56243, 20347.22343]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -328,9 +328,9 @@ public isolated function testMapsWithLong() returns error? { map colors = {"red": 2, "green": 435, "blue": 2034723}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -347,9 +347,9 @@ public isolated function testMapsWithStrings() returns error? { map colors = {"red": "2", "green": "435", "blue": "2034723"}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -366,8 +366,8 @@ public isolated function testMapsWithUnionTypes() returns error? { map colors = {"red": "2", "green": "435", "blue": "2034723"}; Schema avro = check new (schema); - byte[]|Error encodedValue = avro.toAvro(colors); - test:assertTrue(encodedValue is Error); + byte[]|Error serializedValue = avro.toAvro(colors); + test:assertTrue(serializedValue is Error); } @test:Config { @@ -384,9 +384,9 @@ public isolated function testMapsWithBoolean() returns error? { map colors = {"red": true, "green": false, "blue": false}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -403,9 +403,9 @@ public isolated function testMapsWithBooleanWithReadOnlyValues() returns error? map & readonly colors = {"red": true, "green": false, "blue": false}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map & readonly deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map & readonly deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -429,9 +429,9 @@ public isolated function testMapsWithMaps() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map> deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map> deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -458,9 +458,9 @@ public isolated function testMapsWithNestedMapsWithReadOnlyValues() returns erro }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map>> & readonly deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map>> & readonly deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -487,9 +487,9 @@ public isolated function testMapsWithNestedMaps() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map>> deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(colors, deserialize); + byte[] serializedValue = check avro.toAvro(colors); + map>> deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -509,9 +509,9 @@ public isolated function testMapsWithLongArray() returns error? { map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -531,9 +531,9 @@ public isolated function testMapsWithIntArray() returns error? { map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -553,9 +553,9 @@ public isolated function testMapsWithIntArrayWithReadOnlyValues() returns error? map & readonly colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map & readonly deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map & readonly deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -575,9 +575,9 @@ public isolated function testMapsWithFloatArray() returns error? { map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -597,9 +597,9 @@ public isolated function testMapsWithStringArray() returns error? { map colors = {"red": ["252", "122", "41"], "green": ["235", "163", "23"], "blue": ["207", "123"]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -619,9 +619,9 @@ public isolated function testMapsWithUnionArray() returns error? { map colors = {"red": ["252", "122", "41"], "green": ["235", "163", "23"], "blue": ["207", "123"]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -641,9 +641,9 @@ public isolated function testMapsWithUnionIntArray() returns error? { map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -663,9 +663,9 @@ public isolated function testMapsWithUnionLongArray() returns error? { map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -685,9 +685,9 @@ public isolated function testMapsWithUnionFloatArray() returns error? { map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -707,9 +707,9 @@ public isolated function testMapsWithUnionDoubleArray() returns error? { map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -729,9 +729,9 @@ public isolated function testMapsWithBytesArray() returns error? { map colors = {"red": ["252".toBytes(), "122".toBytes(), "41".toBytes()], "green": ["235".toBytes(), "163".toBytes(), "23".toBytes()], "blue": ["207".toBytes(), "123".toBytes()]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -755,9 +755,9 @@ public isolated function testMapsWithFixedArray() returns error? { map colors = {"red": ["252".toBytes(), "122".toBytes(), "411".toBytes()], "green": ["235".toBytes(), "163".toBytes(), "213".toBytes()], "blue": ["207".toBytes(), "123".toBytes()]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -777,9 +777,9 @@ public isolated function testMapsWithBooleanArray() returns error? { map colors = {"red": [true, false, true], "green": [false, true, false], "blue": [true, false]}; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, colors); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, colors); } @test:Config { @@ -796,9 +796,9 @@ public isolated function testMapsWithArrayOfRecordArray() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(instructors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(instructors, deserialize); + byte[] serializedValue = check avro.toAvro(instructors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(instructors, deserializedValue); } @test:Config { @@ -828,8 +828,8 @@ public isolated function testArrayOfStringArrayMaps() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(colors); + map deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, colors); } @@ -858,9 +858,9 @@ public isolated function testMapsWithNestedRecordMaps() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(lecturers); - map> deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(lecturers, deserialize); + byte[] serializedValue = check avro.toAvro(lecturers); + map> deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(lecturers, deserializedValue); } @test:Config { @@ -880,9 +880,9 @@ public isolated function testMapsWithNestedRecordArrayMaps() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(lecturers); - map> deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(lecturers, deserialize); + byte[] serializedValue = check avro.toAvro(lecturers); + map> deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(lecturers, deserializedValue); } @test:Config { @@ -899,9 +899,9 @@ public isolated function testMapsWithRecordArray() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(instructors); - map deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(instructors, deserialize); + byte[] serializedValue = check avro.toAvro(instructors); + map deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(instructors, deserializedValue); } @test:Config { @@ -922,9 +922,9 @@ public isolated function testMapsWithNestedRecordArrayReadOnlyMaps() returns err }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(lecturers); - map> & readonly deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(lecturers, deserialize); + byte[] serializedValue = check avro.toAvro(lecturers); + map> & readonly deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(lecturers, deserializedValue); } // @test:Config { @@ -994,7 +994,7 @@ public isolated function testMapsWithNestedRecordArrayReadOnlyMaps() returns err // }; // Schema avro = check new (schema); -// byte[] encodedValue = check avro.toAvro(lecturers); -// map> deserialize = check avro.fromAvro(encodedValue); -// test:assertEquals(lecturers, deserialize); +// byte[] serializedValue = check avro.toAvro(lecturers); +// map> deserializedValue = check avro.fromAvro(serializedValue); +// test:assertEquals(lecturers, deserializedValue); // } diff --git a/ballerina/tests/primitive_tests.bal b/ballerina/tests/primitive_tests.bal index cc22356..2397304 100644 --- a/ballerina/tests/primitive_tests.bal +++ b/ballerina/tests/primitive_tests.bal @@ -30,8 +30,8 @@ public isolated function testIntValue() returns error? { int value = 5; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(value); - int deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(value); + int deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, value); } @@ -49,8 +49,8 @@ public isolated function testFloatValue() returns error? { float value = 5.5; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(value); - float deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(value); + float deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, value); } @@ -68,8 +68,8 @@ public isolated function testDoubleValue() returns error? { float value = 5.5595; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(value); - float deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(value); + float deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, value); } @@ -86,8 +86,8 @@ public isolated function testLongValue() returns error? { int value = 555950000000000000; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(value); - int deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(value); + int deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, value); } @@ -104,8 +104,8 @@ public isolated function testStringValue() returns error? { string value = "test"; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(value); - string deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(value); + string deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, value); } @@ -122,8 +122,8 @@ public isolated function testBoolean() returns error? { boolean value = true; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(value); - boolean deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(value); + boolean deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, value); } @@ -139,8 +139,8 @@ public isolated function testNullValues() returns error? { }`; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(()); - () deserializedValue = check avro.fromAvro(encodedValue); + byte[] serializedValue = check avro.toAvro(()); + () deserializedValue = check avro.fromAvro(serializedValue); test:assertEquals(deserializedValue, ()); } @@ -156,6 +156,6 @@ public isolated function testNullValuesWithNonNullData() returns error? { }`; Schema avro = check new (schema); - byte[]|error encodedValue = avro.toAvro("string"); - test:assertTrue(encodedValue is error); + byte[]|error serializedValue = avro.toAvro("string"); + test:assertTrue(serializedValue is error); } diff --git a/ballerina/tests/record_tests.bal b/ballerina/tests/record_tests.bal index af7a66d..c369749 100644 --- a/ballerina/tests/record_tests.bal +++ b/ballerina/tests/record_tests.bal @@ -38,8 +38,8 @@ public isolated function testRecords() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(student); - Student deserialize = check avro.fromAvro(serialize); - test:assertEquals(student, deserialize); + Student deserializedValue = check avro.fromAvro(serialize); + test:assertEquals(student, deserializedValue); } @test:Config { @@ -63,9 +63,9 @@ public isolated function testRecordsWithDifferentTypeOfFields() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(student); - Person deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(student, deserialize); + byte[] serializedValue = check avro.toAvro(student); + Person deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(student, deserializedValue); } @test:Config { @@ -137,9 +137,9 @@ public isolated function testNestedRecords() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(lecturer); - Lecturer3 deserialize = check avro.fromAvro(serialize); + Lecturer3 deserializedValue = check avro.fromAvro(serialize); // deserialize.instructor.student.name = "Sam"; - test:assertEquals(deserialize, lecturer); + test:assertEquals(deserializedValue, lecturer); } @test:Config { @@ -164,8 +164,8 @@ public isolated function testArraysInRecords() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(colors); - Color deserialize = check avro.fromAvro(serialize); - test:assertEquals(colors, deserialize); + Color deserializedValue = check avro.fromAvro(serialize); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -190,8 +190,8 @@ public isolated function testArraysInReadOnlyRecords() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(colors); - Color & readonly deserialize = check avro.fromAvro(serialize); - test:assertEquals(colors, deserialize); + Color & readonly deserializedValue = check avro.fromAvro(serialize); + test:assertEquals(colors, deserializedValue); } @test:Config { @@ -227,8 +227,8 @@ public isolated function testArraysInRecordsWithInvalidSchema() returns error? { ] }`; Schema avroConsumer = check new (schema2); - Color1|Error deserialize = avroConsumer.fromAvro(serialize); - test:assertTrue(deserialize is Error); + Color1|Error deserializedValue = avroConsumer.fromAvro(serialize); + test:assertTrue(deserializedValue is Error); } @test:Config { @@ -286,8 +286,8 @@ public isolated function testRecordsWithStringRecordUnionType() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(course); - MultipleUnionRecord deserialize = check avro.fromAvro(serialize); - test:assertEquals(deserialize, course); + MultipleUnionRecord deserializedValue = check avro.fromAvro(serialize); + test:assertEquals(deserializedValue, course); } @test:Config { @@ -341,8 +341,8 @@ public isolated function testRecordsWithUnionTypes() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(course); - UnionRecord deserialize = check avro.fromAvro(serialize); - test:assertEquals(deserialize, course); + UnionRecord deserializedValue = check avro.fromAvro(serialize); + test:assertEquals(deserializedValue, course); } @test:Config { @@ -366,9 +366,9 @@ public isolated function testRecordsWithIntFields() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(student); - Person deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(student, deserialize); + byte[] serializedValue = check avro.toAvro(student); + Person deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(student, deserializedValue); } @test:Config { @@ -392,9 +392,9 @@ public isolated function testRecordsWithLongFields() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(student); - Person deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(student, deserialize); + byte[] serializedValue = check avro.toAvro(student); + Person deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(student, deserializedValue); } @test:Config { @@ -418,9 +418,9 @@ public isolated function testRecordsWithFloatFields() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(student); - Students deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(student, deserialize); + byte[] serializedValue = check avro.toAvro(student); + Students deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(student, deserializedValue); } @test:Config { @@ -444,9 +444,9 @@ public isolated function testRecordsWithDoubleFields() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(student); - Students deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(student, deserialize); + byte[] serializedValue = check avro.toAvro(student); + Students deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(student, deserializedValue); } @test:Config { @@ -470,9 +470,9 @@ public isolated function testRecordsWithBooleanFields() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(student); - StudentRec deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(student, deserialize); + byte[] serializedValue = check avro.toAvro(student); + StudentRec deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(student, deserializedValue); } @test:Config { @@ -556,8 +556,8 @@ public isolated function testOptionalValuesInRecords() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(lecturer5); - Lecturer5 deserialize = check avro.fromAvro(serialize); - test:assertEquals(deserialize, lecturer5); + Lecturer5 deserializedValue = check avro.fromAvro(serialize); + test:assertEquals(deserializedValue, lecturer5); } @test:Config { @@ -689,6 +689,6 @@ public isolated function testOptionalMultipleFieldsInRecords() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(lecturer6); - Lecturer6 deserialize = check avro.fromAvro(serialize); - test:assertEquals(deserialize, lecturer6); + Lecturer6 deserializedValue = check avro.fromAvro(serialize); + test:assertEquals(deserializedValue, lecturer6); } diff --git a/ballerina/tests/test.bal b/ballerina/tests/test.bal index 6977eea..f377e16 100644 --- a/ballerina/tests/test.bal +++ b/ballerina/tests/test.bal @@ -44,9 +44,9 @@ public isolated function testUnionEnums() returns error? { field1: ONE }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(number); - UnionEnumRecord deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(number, deserialize); + byte[] serializedValue = check avro.toAvro(number); + UnionEnumRecord deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(number, deserializedValue); } @test:Config { @@ -60,9 +60,9 @@ public isolated function testUnionFixed() returns error? { field1: "ON".toBytes() }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(number); - UnionFixedRecord deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(number, deserialize); + byte[] serializedValue = check avro.toAvro(number); + UnionFixedRecord deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(number, deserializedValue); } @test:Config { @@ -76,9 +76,9 @@ public isolated function testUnionFixeWithReadOnlyValues() returns error? { field1: "ON".toBytes().cloneReadOnly() }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(number); - UnionFixedRecord & readonly deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(number, deserialize); + byte[] serializedValue = check avro.toAvro(number); + UnionFixedRecord & readonly deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(number, deserializedValue); } @test:Config { @@ -95,9 +95,9 @@ public isolated function testUnionsWithRecordsAndStrings() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(number); - UnionRec deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(number, deserialize); + byte[] serializedValue = check avro.toAvro(number); + UnionRec deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(number, deserializedValue); } @test:Config { @@ -114,9 +114,9 @@ public isolated function testUnionsWithReadOnlyRecords() returns error? { }; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(number); - ReadOnlyRec deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(number, deserialize); + byte[] serializedValue = check avro.toAvro(number); + ReadOnlyRec deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(number, deserializedValue); } @test:Config { @@ -134,9 +134,9 @@ public isolated function testEnums() returns error? { Numbers number = "ONE"; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(number); - Numbers deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(number, deserialize); + byte[] serializedValue = check avro.toAvro(number); + Numbers deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(number, deserializedValue); } @test:Config { @@ -154,9 +154,9 @@ public isolated function testEnumsWithReadOnlyValues() returns error? { Numbers & readonly number = "ONE"; Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(number); - Numbers & readonly deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(number, deserialize); + byte[] serializedValue = check avro.toAvro(number); + Numbers & readonly deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(number, deserializedValue); } @test:Config { @@ -174,8 +174,8 @@ public isolated function testEnumsWithString() returns error? { string number = "FIVE"; Schema avro = check new (schema); - byte[]|Error encodedValue = avro.toAvro(number); - test:assertTrue(encodedValue is Error); + byte[]|Error serializedValue = avro.toAvro(number); + test:assertTrue(serializedValue is Error); } @test:Config { @@ -192,8 +192,8 @@ public isolated function testFixedWithInvalidSize() returns error? { byte[] value = "u00".toBytes(); Schema avro = check new (schema); - byte[]|Error encodedValue = avro.toAvro(value); - test:assertTrue(encodedValue is Error); + byte[]|Error serializedValue = avro.toAvro(value); + test:assertTrue(serializedValue is Error); } @test:Config { @@ -210,9 +210,9 @@ public isolated function testFixed() returns error? { byte[] value = "u00ffffffffffffx".toBytes(); Schema avro = check new (schema); - byte[] encodedValue = check avro.toAvro(value); - byte[] deserialize = check avro.fromAvro(encodedValue); - test:assertEquals(deserialize, value); + byte[] serializedValue = check avro.toAvro(value); + byte[] deserializedValue = check avro.fromAvro(serializedValue); + test:assertEquals(deserializedValue, value); } @test:Config { @@ -240,8 +240,8 @@ public function testDbSchemaWithRecords() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(changeKey); - SchemaChangeKey deserialize = check avro.fromAvro(serialize); - test:assertEquals(changeKey, deserialize); + SchemaChangeKey deserializedValue = check avro.fromAvro(serialize); + test:assertEquals(changeKey, deserializedValue); } @@ -316,8 +316,8 @@ public function testComplexDbSchema() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(envelope); - Envelope deserialize = check avro.fromAvro(serialize); - test:assertEquals(deserialize, envelope); + Envelope deserializedValue = check avro.fromAvro(serialize); + test:assertEquals(deserializedValue, envelope); } @test:Config { @@ -441,6 +441,6 @@ public function testComplexDbSchemaWithNestedRecords() returns error? { Schema avro = check new (schema); byte[] serialize = check avro.toAvro(envelope2); - Envelope2 deserialize = check avro.fromAvro(serialize); - test:assertEquals(deserialize, envelope2); + Envelope2 deserializedValue = check avro.fromAvro(serialize); + test:assertEquals(deserializedValue, envelope2); } From 5e368259ef13aff4a11e4dd781f021468d17ae84 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Fri, 10 May 2024 16:08:37 +0530 Subject: [PATCH 43/47] Move redundant codes in test cases to a separate method --- ballerina/tests/array_tests.bal | 103 +---- ballerina/tests/byte_tests.bal | 25 +- ballerina/tests/map_tests.bal | 316 ++------------- ballerina/tests/primitive_tests.bal | 39 +- ballerina/tests/record_tests.bal | 378 ++---------------- ...chema_record_multiple_optional_fields.json | 95 +++++ .../tests/resources/schema_record_nested.json | 49 +++ .../schema_record_optional_fields.json | 54 +++ .../schema_record_string_record_union.json | 40 ++ .../tests/resources/schema_record_union.json | 36 ++ ballerina/tests/test.bal | 88 +--- ballerina/tests/types.bal | 64 +++ 12 files changed, 458 insertions(+), 829 deletions(-) create mode 100644 ballerina/tests/resources/schema_record_multiple_optional_fields.json create mode 100644 ballerina/tests/resources/schema_record_nested.json create mode 100644 ballerina/tests/resources/schema_record_optional_fields.json create mode 100644 ballerina/tests/resources/schema_record_string_record_union.json create mode 100644 ballerina/tests/resources/schema_record_union.json diff --git a/ballerina/tests/array_tests.bal b/ballerina/tests/array_tests.bal index 509e71a..50aa893 100644 --- a/ballerina/tests/array_tests.bal +++ b/ballerina/tests/array_tests.bal @@ -30,11 +30,7 @@ public isolated function testIntArrays() returns error? { }`; int[] numbers = [22, 556, 78]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(numbers); - int[] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, numbers); + return verifyOperation(IntArray, numbers, schema); } @test:Config { @@ -50,11 +46,7 @@ public isolated function testReadOnlyIntArrays() returns error? { }`; int[] & readonly numbers = [22, 556, 78]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(numbers); - int[] & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, numbers); + return verifyOperation(ReadOnlyIntArray, numbers, schema); } @test:Config { @@ -70,11 +62,7 @@ public isolated function testStringArrays() returns error? { }`; string[] colors = ["red", "green", "blue"]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - string[] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(StringArray, colors, schema); } @test:Config { @@ -89,12 +77,8 @@ public isolated function testReadOnlyStringArrays() returns error? { "items": "string" }`; - string[] & readonly colors = ["red", "green", "blue"]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - string[] & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + ReadOnlyStringArray colors = ["red", "green", "blue"]; + return verifyOperation(ReadOnlyStringArray, colors, schema); } @test:Config { @@ -115,11 +99,7 @@ public isolated function testArrayOfStringArrays() returns error? { }`; string[][] colors = [["red", "green", "blue"]]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - string[][] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(String2DArray, colors, schema); } @test:Config { @@ -140,11 +120,7 @@ public isolated function testReadOnlyArrayOfStringArrays() returns error? { }`; string[][] & readonly colors = [["red", "green", "blue"]]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - string[][] & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(ReadOnlyString2DArray, colors, schema); } @test:Config { @@ -164,11 +140,7 @@ public isolated function testEnumArrays() returns error? { }`; Numbers[] colors = ["ONE", "TWO", "THREE"]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - Numbers[] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(EnumArray, colors, schema); } @test:Config { @@ -193,11 +165,7 @@ public isolated function testArrayOfEnumArrays() returns error? { }`; Numbers[][] colors = [["ONE", "TWO", "THREE"], ["ONE", "TWO", "THREE"]]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - Numbers[][] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(Enum2DArray, colors, schema); } @test:Config { @@ -213,11 +181,7 @@ public isolated function testFloatArrays() returns error? { }`; float[] numbers = [22.4, 556.84350, 78.0327]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(numbers); - float[] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, numbers); + return verifyOperation(FloatArray, numbers, schema); } @test:Config { @@ -233,11 +197,7 @@ public isolated function testDoubleArrays() returns error? { }`; float[] numbers = [22.439475948, 556.843549485340, 78.032985693457]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(numbers); - float[] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, numbers); + return verifyOperation(FloatArray, numbers, schema); } @test:Config { @@ -253,11 +213,7 @@ public isolated function testLongArrays() returns error? { }`; int[] numbers = [223432, 55423326, 7823423]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(numbers); - int[] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, numbers); + return verifyOperation(IntArray, numbers, schema); } @test:Config { @@ -292,11 +248,7 @@ public isolated function testBooleanArrays() returns error? { }`; boolean[] numbers = [true, true, false]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(numbers); - boolean[] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, numbers); + return verifyOperation(BooleanArray, numbers, schema); } @test:Config { @@ -334,11 +286,7 @@ public isolated function testArraysWithFixed() returns error? { }`; byte[][] numbers = ["22".toBytes(), "55".toBytes(), "78".toBytes()]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(numbers); - byte[][] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, numbers); + return verifyOperation(ArrayOfByteArray, numbers, schema); } @test:Config { @@ -355,11 +303,7 @@ public isolated function testRecordsInArrays() returns error? { name: "John", subject: "math" }]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(students); - Student[] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, students); + return verifyOperation(StudentArray, students, schema); } @test:Config { @@ -376,11 +320,7 @@ public isolated function testRecordsInReadOnlyArrays() returns error? { name: "John", subject: "math" }]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(students); - Student[] & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, students); + return verifyOperation(ReadOnlyStudentArray, students, schema); } @test:Config { @@ -403,11 +343,7 @@ public isolated function testRecordArraysInArrays() returns error? { name: "John", subject: "math" }]]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(students); - Student[][] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, students); + return verifyOperation(Student2DArray, students, schema); } @test:Config { @@ -431,8 +367,5 @@ public isolated function testRecordArraysInReadOnlyArrays() returns error? { subject: "math" }]]; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(students); - Student[][] & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, students); + return verifyOperation(ReadOnlyStudent2DArray, students, schema); } diff --git a/ballerina/tests/byte_tests.bal b/ballerina/tests/byte_tests.bal index b7f40a3..5c7ab38 100644 --- a/ballerina/tests/byte_tests.bal +++ b/ballerina/tests/byte_tests.bal @@ -36,11 +36,7 @@ public isolated function testRecordsWithBytes() returns error? { name: "Liam", favorite_color: "yellow".toBytes() }; - - Schema avro = check new(schema); - byte[] serializedValue = check avro.toAvro(student); - Student1 deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, student); + return verifyOperation(Student1, student, schema); } @test:Config { @@ -56,11 +52,7 @@ public isolated function testArraysWithBytes() returns error? { }`; byte[][] numbers = ["22.4".toBytes(), "556.84350".toBytes(), "78.0327".toBytes()]; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(numbers); - byte[][] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, numbers); + return verifyOperation(ArrayOfByteArray, numbers, schema); } @test:Config { @@ -75,11 +67,7 @@ public isolated function testBytes() returns error? { }`; byte[] value = "5".toBytes(); - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(value); - byte[] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, value); + return verifyOperation(ByteArray, value, schema); } @test:Config { @@ -100,10 +88,5 @@ public isolated function testNestedRecordsWithBytes() returns error? { byteData: "ddd".toBytes().cloneReadOnly() } }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(lecturer4); - Lecturer4 deserializedValue = check avro.fromAvro(serialize); - // deserialize.instructor.student.name = "Sam"; - test:assertEquals(deserializedValue, lecturer4); + return verifyOperation(Lecturer4, lecturer4, schema); } diff --git a/ballerina/tests/map_tests.bal b/ballerina/tests/map_tests.bal index f756a11..5a9d553 100644 --- a/ballerina/tests/map_tests.bal +++ b/ballerina/tests/map_tests.bal @@ -29,10 +29,7 @@ public isolated function testMapsWithBytes() returns error? { }`; map colors = {"red": "0".toBytes(), "green": "1".toBytes(), "blue": "2".toBytes()}; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(ByteArrayMap, colors, schema); } @test:Config { @@ -51,10 +48,7 @@ public isolated function testMapsWithFixed() returns error? { }`; map colors = {"red": "0".toBytes(), "green": "1".toBytes(), "blue": "2".toBytes()}; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(ByteArrayMap, colors, schema); } @test:Config { @@ -81,10 +75,7 @@ public isolated function testMapsOfFixedMaps() returns error? { "green": {"r": "0".toBytes(), "g": "1".toBytes(), "b": "2".toBytes()}, "blue": {"r": "0".toBytes(), "g": "1".toBytes(), "b": "2".toBytes()} }; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map> deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(MapOfByteArrayMap, colors, schema); } @test:Config { @@ -100,10 +91,7 @@ public isolated function testReadOnlyMapsWithReadOnlyRecords() returns error? { "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} }; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(instructors); - map & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(instructors, deserializedValue); + return verifyOperation(ReadOnlyMapOfReadOnlyRecord, instructors, schema); } @test:Config { @@ -118,11 +106,7 @@ public isolated function testMapsWithRecordsWithReadOnly() returns error? { "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(instructors); - map & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(instructors, deserializedValue); + return verifyOperation(ReadOnlyMapOfRecord, instructors, schema); } @test:Config { @@ -137,11 +121,7 @@ public isolated function testMapsWithReadOnlyRecordsWithReadOnly() returns error "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(instructors); - map & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(instructors, deserializedValue); + return verifyOperation(ReadOnlyMapOfReadOnlyRecord, instructors, schema); } @test:Config { @@ -156,11 +136,7 @@ public isolated function testMapsWithReadOnlyRecords() returns error? { "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(instructors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(instructors, deserializedValue); + return verifyOperation(ReadOnlyMapOfRecord, instructors, schema); } @test:Config { @@ -175,11 +151,7 @@ public isolated function testMapsWithRecords() returns error? { "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(instructors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(instructors, deserializedValue); + return verifyOperation(RecordMap, instructors, schema); } @test:Config { @@ -194,15 +166,11 @@ public isolated function testMapsWithInt() returns error? { }`; map colors = {"red": 0, "green": 1, "blue": 2}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(IntMap, colors, schema); } @test:Config { - groups: ["map", "enum", "lk"] + groups: ["map", "enum"] } public isolated function testMapsWithEnum() returns error? { string schema = string ` @@ -217,11 +185,7 @@ public isolated function testMapsWithEnum() returns error? { }`; map colors = {"red": "ONE", "green": "TWO", "blue": "THREE"}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(EnumMap, colors, schema); } @test:Config { @@ -245,11 +209,7 @@ public isolated function testMapsWithEnumArrays() returns error? { }`; map colors = {"red": ["ONE", "TWO", "THREE"], "green": ["ONE", "TWO", "THREE"], "blue": ["ONE", "TWO", "THREE"]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(EnumArrayMap, colors, schema); } @test:Config { @@ -264,11 +224,7 @@ public isolated function testMapsWithFloat() returns error? { }`; map colors = {"red": 2.3453, "green": 435.563, "blue": 20347.23}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(FloatMap, colors, schema); } @test:Config { @@ -283,11 +239,7 @@ public isolated function testMapsWithDouble() returns error? { }`; map colors = {"red": 2.3453, "green": 435.563, "blue": 20347.23}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(FloatMap, colors, schema); } @test:Config { @@ -307,11 +259,7 @@ public isolated function testMapsWithDoubleArray() returns error? { }`; map colors = {"red": [2.3434253, 435.56433, 20347.22343], "green": [2.3452343, 435.56343, 20347.2423], "blue": [2.3453243, 435.56243, 20347.22343]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(FloatArrayMap, colors, schema); } @test:Config { @@ -326,11 +274,7 @@ public isolated function testMapsWithLong() returns error? { }`; map colors = {"red": 2, "green": 435, "blue": 2034723}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(IntMap, colors, schema); } @test:Config { @@ -345,11 +289,7 @@ public isolated function testMapsWithStrings() returns error? { }`; map colors = {"red": "2", "green": "435", "blue": "2034723"}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(StringMap, colors, schema); } @test:Config { @@ -371,7 +311,7 @@ public isolated function testMapsWithUnionTypes() returns error? { } @test:Config { - groups: ["map", "boolean", "ssq"] + groups: ["map", "boolean"] } public isolated function testMapsWithBoolean() returns error? { string schema = string ` @@ -382,11 +322,7 @@ public isolated function testMapsWithBoolean() returns error? { }`; map colors = {"red": true, "green": false, "blue": false}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(BooleanMap, colors, schema); } @test:Config { @@ -401,11 +337,7 @@ public isolated function testMapsWithBooleanWithReadOnlyValues() returns error? }`; map & readonly colors = {"red": true, "green": false, "blue": false}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(ReadOnlyBooleanMap, colors, schema); } @test:Config { @@ -427,11 +359,7 @@ public isolated function testMapsWithMaps() returns error? { "green": {"r": 5, "g": 6, "b": 7}, "blue": {"r": 8, "g": 9, "b": 10} }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map> deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(MapOfIntMap, colors, schema); } @test:Config { @@ -456,11 +384,7 @@ public isolated function testMapsWithNestedMapsWithReadOnlyValues() returns erro "green": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}}, "blue": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}} }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map>> & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(ReadOnlyMapOfReadOnlyMap, colors, schema); } @test:Config { @@ -485,11 +409,7 @@ public isolated function testMapsWithNestedMaps() returns error? { "green": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}}, "blue": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}} }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map>> deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(colors, deserializedValue); + return verifyOperation(MapOfMap, colors, schema); } @test:Config { @@ -507,11 +427,7 @@ public isolated function testMapsWithLongArray() returns error? { }`; map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(IntArrayMap, colors, schema); } @test:Config { @@ -529,15 +445,11 @@ public isolated function testMapsWithIntArray() returns error? { }`; map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(IntArrayMap, colors, schema); } @test:Config { - groups: ["map", "int", "az"] + groups: ["map", "int"] } public isolated function testMapsWithIntArrayWithReadOnlyValues() returns error? { string schema = string ` @@ -551,11 +463,7 @@ public isolated function testMapsWithIntArrayWithReadOnlyValues() returns error? }`; map & readonly colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(ReadOnlyIntArrayMap, colors, schema); } @test:Config { @@ -573,11 +481,7 @@ public isolated function testMapsWithFloatArray() returns error? { }`; map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(FloatArrayMap, colors, schema); } @test:Config { @@ -595,11 +499,7 @@ public isolated function testMapsWithStringArray() returns error? { }`; map colors = {"red": ["252", "122", "41"], "green": ["235", "163", "23"], "blue": ["207", "123"]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(StringArrayMap, colors, schema); } @test:Config { @@ -617,11 +517,7 @@ public isolated function testMapsWithUnionArray() returns error? { }`; map colors = {"red": ["252", "122", "41"], "green": ["235", "163", "23"], "blue": ["207", "123"]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(StringArrayMap, colors, schema); } @test:Config { @@ -639,11 +535,7 @@ public isolated function testMapsWithUnionIntArray() returns error? { }`; map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(IntArrayMap, colors, schema); } @test:Config { @@ -661,11 +553,7 @@ public isolated function testMapsWithUnionLongArray() returns error? { }`; map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(IntArrayMap, colors, schema); } @test:Config { @@ -683,11 +571,7 @@ public isolated function testMapsWithUnionFloatArray() returns error? { }`; map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(FloatArrayMap, colors, schema); } @test:Config { @@ -705,11 +589,7 @@ public isolated function testMapsWithUnionDoubleArray() returns error? { }`; map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(FloatArrayMap, colors, schema); } @test:Config { @@ -727,11 +607,7 @@ public isolated function testMapsWithBytesArray() returns error? { }`; map colors = {"red": ["252".toBytes(), "122".toBytes(), "41".toBytes()], "green": ["235".toBytes(), "163".toBytes(), "23".toBytes()], "blue": ["207".toBytes(), "123".toBytes()]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(ByteArrayArrayMap, colors, schema); } @test:Config { @@ -753,11 +629,7 @@ public isolated function testMapsWithFixedArray() returns error? { }`; map colors = {"red": ["252".toBytes(), "122".toBytes(), "411".toBytes()], "green": ["235".toBytes(), "163".toBytes(), "213".toBytes()], "blue": ["207".toBytes(), "123".toBytes()]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(ByteArrayArrayMap, colors, schema); } @test:Config { @@ -775,11 +647,7 @@ public isolated function testMapsWithBooleanArray() returns error? { }`; map colors = {"red": [true, false, true], "green": [false, true, false], "blue": [true, false]}; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(BooleanArrayMap, colors, schema); } @test:Config { @@ -794,11 +662,7 @@ public isolated function testMapsWithArrayOfRecordArray() returns error? { "doe": [[{name: "Doe", student: {name: "Bob", subject: "Science"}}, {name: "Doe", student: {name: "Bob", subject: "Science"}}]], "jane": [[{name: "Jane", student: {name: "Charlie", subject: "English"}}, {name: "Jane", student: {name: "Charlie", subject: "English"}}]] }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(instructors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(instructors, deserializedValue); + return verifyOperation(MapOfArrayOfRecordArray, instructors, schema); } @test:Config { @@ -826,11 +690,7 @@ public isolated function testArrayOfStringArrayMaps() returns error? { "green": [["red", "green", "blue"], ["red", "green", "blue"], ["red", "green", "blue"]], "blue": [["red", "green", "blue"], ["red", "green", "blue"], ["red", "green", "blue"]] }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(colors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, colors); + return verifyOperation(StringArrayArrayMap, colors, schema); } @test:Config { @@ -856,11 +716,7 @@ public isolated function testMapsWithNestedRecordMaps() returns error? { "doe": {"john": lec, "doe": lec}, "jane": {"john": lec, "doe": lec} }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(lecturers); - map> deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(lecturers, deserializedValue); + return verifyOperation(MapOfRecordMap, lecturers, schema); } @test:Config { @@ -878,11 +734,7 @@ public isolated function testMapsWithNestedRecordArrayMaps() returns error? { "doe": {"r": lecs, "g": lecs, "b": lecs}, "jane": {"r": lecs, "g": lecs, "b": lecs} }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(lecturers); - map> deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(lecturers, deserializedValue); + return verifyOperation(MapOfRecordArrayMap, lecturers, schema); } @test:Config { @@ -897,11 +749,7 @@ public isolated function testMapsWithRecordArray() returns error? { "doe": [{name: "Doe", student: {name: "Bob", subject: "Science"}}, {name: "Doe", student: {name: "Bob", subject: "Science"}}], "jane": [{name: "Jane", student: {name: "Charlie", subject: "English"}}, {name: "Jane", student: {name: "Charlie", subject: "English"}}] }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(instructors); - map deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(instructors, deserializedValue); + return verifyOperation(MapOfRecordArray, instructors, schema); } @test:Config { @@ -920,81 +768,5 @@ public isolated function testMapsWithNestedRecordArrayReadOnlyMaps() returns err "doe": mapValue.cloneReadOnly(), "jane": mapValue.cloneReadOnly() }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(lecturers); - map> & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(lecturers, deserializedValue); -} - -// @test:Config { -// groups: ["map", "record"] -// } -// public isolated function testMapsWithReadOnlyRecordArrayReadOnlyMaps() returns error? { -// string schema = string ` -// { -// "type": "map", -// "values": { -// "type": "map", -// "values": { -// "type": "array", -// "items": { -// "type": "record", -// "name": "Lecturer", -// "fields": [ -// { -// "name": "name", -// "type": ["null", "string"] -// }, -// { -// "name": "instructor", -// "type": ["null", { -// "type": "record", -// "name": "Instructor", -// "fields": [ -// { -// "name": "name", -// "type": ["null", "string"] -// }, -// { -// "name": "student", -// "type": ["null", { -// "type": "record", -// "name": "Student", -// "fields": [ -// { -// "name": "name", -// "type": ["null", "string"] -// }, -// { -// "name": "subject", -// "type": ["null", "string"] -// } -// ] -// }] -// } -// ] -// }] -// } -// ] -// } -// } -// } -// }`; - - -// Lecturer[] & readonly lecs = [{name: "John", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}, -// {name: "Doe", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}]; - -// map mapValue = {"r": lecs, "g": lecs, "b": lecs}; -// map> lecturers = { -// "john": mapValue, -// "doe": mapValue, -// "jane": mapValue -// }; - -// Schema avro = check new (schema); -// byte[] serializedValue = check avro.toAvro(lecturers); -// map> deserializedValue = check avro.fromAvro(serializedValue); -// test:assertEquals(lecturers, deserializedValue); -// } + return verifyOperation(ReadOnlyMapOfRecordArray, lecturers, schema); +} diff --git a/ballerina/tests/primitive_tests.bal b/ballerina/tests/primitive_tests.bal index 2397304..0b7add8 100644 --- a/ballerina/tests/primitive_tests.bal +++ b/ballerina/tests/primitive_tests.bal @@ -28,11 +28,7 @@ public isolated function testIntValue() returns error? { }`; int value = 5; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(value); - int deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, value); + return verifyOperation(int, value, schema); } @test:Config { @@ -47,11 +43,7 @@ public isolated function testFloatValue() returns error? { }`; float value = 5.5; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(value); - float deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, value); + return verifyOperation(float, value, schema); } @test:Config { @@ -66,11 +58,7 @@ public isolated function testDoubleValue() returns error? { }`; float value = 5.5595; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(value); - float deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, value); + return verifyOperation(float, value, schema); } @test:Config { @@ -85,10 +73,7 @@ public isolated function testLongValue() returns error? { }`; int value = 555950000000000000; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(value); - int deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, value); + return verifyOperation(int, value, schema); } @test:Config { @@ -103,10 +88,7 @@ public isolated function testStringValue() returns error? { }`; string value = "test"; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(value); - string deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, value); + return verifyOperation(string, value, schema); } @test:Config { @@ -121,10 +103,7 @@ public isolated function testBoolean() returns error? { }`; boolean value = true; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(value); - boolean deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, value); + return verifyOperation(boolean, value, schema); } @test:Config { @@ -137,11 +116,7 @@ public isolated function testNullValues() returns error? { "name" : "nullValue", "namespace": "data" }`; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(()); - () deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, ()); + return verifyOperation(NullType, (), schema); } @test:Config { diff --git a/ballerina/tests/record_tests.bal b/ballerina/tests/record_tests.bal index c369749..bc26942 100644 --- a/ballerina/tests/record_tests.bal +++ b/ballerina/tests/record_tests.bal @@ -15,6 +15,7 @@ // under the License. import ballerina/test; +import ballerina/io; @test:Config { groups: ["record"] @@ -35,11 +36,7 @@ public isolated function testRecords() returns error? { name: "Liam", subject: "geology" }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(student); - Student deserializedValue = check avro.fromAvro(serialize); - test:assertEquals(student, deserializedValue); + return verifyOperation(Student, student, schema); } @test:Config { @@ -61,67 +58,15 @@ public isolated function testRecordsWithDifferentTypeOfFields() returns error? { name: "Liam", age: 52 }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(student); - Person deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(student, deserializedValue); + return verifyOperation(Person, student, schema); } @test:Config { groups: ["record"] } public isolated function testNestedRecords() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Lecturer", - "fields": [ - { - "name": "name", - "type": { - "type": "map", - "values" : "int", - "default": {} - } - }, - { - "name": "age", - "type": "long" - }, - { - "name": "instructor", - "type": { - "name": "Instructor", - "type": "record", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "student", - "type": { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "subject", - "type": "string" - } - ] - } - } - ] - } - } - ] - }`; + string jsonFileName = string `tests/resources/schema_record_nested.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Lecturer3 lecturer = { name: {"John": 1, "Sam": 2, "Liam": 3}, @@ -135,11 +80,7 @@ public isolated function testNestedRecords() returns error? { } }; - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(lecturer); - Lecturer3 deserializedValue = check avro.fromAvro(serialize); - // deserialize.instructor.student.name = "Sam"; - test:assertEquals(deserializedValue, lecturer); + return verifyOperation(Lecturer3, lecturer, schema); } @test:Config { @@ -162,14 +103,11 @@ public isolated function testArraysInRecords() returns error? { colors: ["maroon", "dark red", "light red"] }; - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(colors); - Color deserializedValue = check avro.fromAvro(serialize); - test:assertEquals(colors, deserializedValue); + return verifyOperation(Color, colors, schema); } @test:Config { - groups: ["record", "array", "aaaa"] + groups: ["record", "array"] } public isolated function testArraysInReadOnlyRecords() returns error? { string schema = string ` @@ -183,15 +121,12 @@ public isolated function testArraysInReadOnlyRecords() returns error? { ] }`; - Color & readonly colors = { + ReadOnlyColors colors = { name: "Red", colors: ["maroon", "dark red", "light red"] }; - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(colors); - Color & readonly deserializedValue = check avro.fromAvro(serialize); - test:assertEquals(colors, deserializedValue); + return verifyOperation(ReadOnlyColors, colors, schema); } @test:Config { @@ -232,50 +167,11 @@ public isolated function testArraysInRecordsWithInvalidSchema() returns error? { } @test:Config { - groups: ["record", "union", "pm"] + groups: ["record", "union"] } public isolated function testRecordsWithStringRecordUnionType() returns error? { - string schema = string ` - { - "type": "record", - "name": "Course", - "namespace": "example.avro", - "fields": [ - { - "name": "name", - "type": ["string", "null"] - }, - { - "name": "value", - "type": "float" - }, - { - "name": "credits", - "type": ["null", "int"] - }, - { - "name": "student", - "type": [ - "null", - { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": ["string", "null"] - }, - { - "name": "subject", - "type": ["string", "null"] - } - ] - }, - "string" - ] - } - ] - }`; + string jsonFileName = string `tests/resources/schema_record_string_record_union.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); MultipleUnionRecord course = { name: "data", @@ -284,53 +180,15 @@ public isolated function testRecordsWithStringRecordUnionType() returns error? { student: string `{name: "Jon", subject: "geo"}adsk` }; - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(course); - MultipleUnionRecord deserializedValue = check avro.fromAvro(serialize); - test:assertEquals(deserializedValue, course); + return verifyOperation(MultipleUnionRecord, course, schema); } @test:Config { groups: ["record", "union"] } public isolated function testRecordsWithUnionTypes() returns error? { - string schema = string ` - { - "type": "record", - "name": "Course", - "namespace": "example.avro", - "fields": [ - { - "name": "name", - "type": ["string", "null"] - }, - { - "name": "value", - "type": "float" - }, - { - "name": "credits", - "type": ["null", "int"] - }, - { - "name": "student", - "type": { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": ["string", "null"] - }, - { - "name": "subject", - "type": ["string", "null"] - } - ] - } - } - ] - }`; + string jsonFileName = string `tests/resources/schema_record_union.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); UnionRecord course = { name: "data", @@ -338,11 +196,7 @@ public isolated function testRecordsWithUnionTypes() returns error? { credits: 5, student: {name: "Jon", subject: "geo"} }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(course); - UnionRecord deserializedValue = check avro.fromAvro(serialize); - test:assertEquals(deserializedValue, course); + return verifyOperation(UnionRecord, course, schema); } @test:Config { @@ -364,11 +218,7 @@ public isolated function testRecordsWithIntFields() returns error? { name: "Liam", age: 52 }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(student); - Person deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(student, deserializedValue); + return verifyOperation(Person, student, schema); } @test:Config { @@ -390,11 +240,7 @@ public isolated function testRecordsWithLongFields() returns error? { name: "Liam", age: 52 }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(student); - Person deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(student, deserializedValue); + return verifyOperation(Person, student, schema); } @test:Config { @@ -416,11 +262,7 @@ public isolated function testRecordsWithFloatFields() returns error? { name: "Liam", age: 52.656 }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(student); - Students deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(student, deserializedValue); + return verifyOperation(Students, student, schema); } @test:Config { @@ -442,11 +284,7 @@ public isolated function testRecordsWithDoubleFields() returns error? { name: "Liam", age: 52.656 }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(student); - Students deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(student, deserializedValue); + return verifyOperation(Students, student, schema); } @test:Config { @@ -468,72 +306,15 @@ public isolated function testRecordsWithBooleanFields() returns error? { name: "Liam", under19: false }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(student); - StudentRec deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(student, deserializedValue); + return verifyOperation(StudentRec, student, schema); } @test:Config { groups: ["record", "union"] } public isolated function testOptionalValuesInRecords() returns error? { - string schema = string ` - { - "type": "record", - "name": "Lecturer5", - "fields": [ - { - "name": "name", - "type": [ - "null", - { - "type": "map", - "values": "int" - } - ] - }, - { - "name": "bytes", - "type": ["null", "bytes"] - }, - { - "name": "instructorClone", - "type": ["null", { - "type": "record", - "name": "Instructor", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "student", - "type": ["null", { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "subject", - "type": "string" - } - ] - }] - } - ] - }] - }, - { - "name": "instructors", - "type": ["null", "Instructor"] - } - ] - }`; + string jsonFileName = string `tests/resources/schema_record_optional_fields.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Instructor instructor = { name: "John", @@ -553,114 +334,15 @@ public isolated function testOptionalValuesInRecords() returns error? { instructorClone: instructor.cloneReadOnly(), instructors: instructor }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(lecturer5); - Lecturer5 deserializedValue = check avro.fromAvro(serialize); - test:assertEquals(deserializedValue, lecturer5); + return verifyOperation(Lecturer5, lecturer5, schema); } @test:Config { groups: ["record", "union"] } public isolated function testOptionalMultipleFieldsInRecords() returns error? { - string schema = string ` - { - "type": "record", - "name": "Lecturer6", - "fields": [ - { - "name": "temporary", - "type": ["null", "boolean"] - }, - { - "name": "map", - "type": [ - "null", - { - "type": "map", - "values": "int" - } - ] - }, - { - "name": "number", - "type": [ - "null", - { - "type": "enum", - "name": "Numbers", - "symbols": [ "ONE", "TWO", "THREE", "FOUR" ] - } - ] - }, - { - "name": "bytes", - "type": ["null", { - "type": "fixed", - "name": "FixedBytes", - "size": 2 - }] - }, - { - "name": "age", - "type": ["null", "long"] - }, - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "floatNumber", - "type": ["null", "float"] - }, - { - "name": "colors", - "type": ["null", { - "type": "array", - "items": { - "type": "enum", - "name": "ColorEnum", - "symbols": ["ONE", "TWO", "THREE"] - } - }] - }, - { - "name": "instructorClone", - "type": ["null", { - "type": "record", - "name": "Instructor", - "fields": [ - { - "name": "name", - "type": ["null", "string"] - }, - { - "name": "student", - "type": ["null", { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "subject", - "type": "string" - } - ] - }] - }] - }] - }, - { - "name": "instructors", - "type": ["null", "Instructor"] - } - ] - }`; - + string jsonFileName = string `tests/resources/schema_record_multiple_optional_fields.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Instructor instructor = { name: "John", student: { @@ -686,9 +368,5 @@ public isolated function testOptionalMultipleFieldsInRecords() returns error? { instructorClone: instructor.cloneReadOnly(), instructors: instructor }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(lecturer6); - Lecturer6 deserializedValue = check avro.fromAvro(serialize); - test:assertEquals(deserializedValue, lecturer6); + return verifyOperation(Lecturer6, lecturer6, schema); } diff --git a/ballerina/tests/resources/schema_record_multiple_optional_fields.json b/ballerina/tests/resources/schema_record_multiple_optional_fields.json new file mode 100644 index 0000000..afc77be --- /dev/null +++ b/ballerina/tests/resources/schema_record_multiple_optional_fields.json @@ -0,0 +1,95 @@ +{ + "type": "record", + "name": "Lecturer6", + "fields": [ + { + "name": "temporary", + "type": ["null", "boolean"] + }, + { + "name": "map", + "type": [ + "null", + { + "type": "map", + "values": "int" + } + ] + }, + { + "name": "number", + "type": [ + "null", + { + "type": "enum", + "name": "Numbers", + "symbols": [ "ONE", "TWO", "THREE", "FOUR" ] + } + ] + }, + { + "name": "bytes", + "type": ["null", { + "type": "fixed", + "name": "FixedBytes", + "size": 2 + }] + }, + { + "name": "age", + "type": ["null", "long"] + }, + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "floatNumber", + "type": ["null", "float"] + }, + { + "name": "colors", + "type": ["null", { + "type": "array", + "items": { + "type": "enum", + "name": "ColorEnum", + "symbols": ["ONE", "TWO", "THREE"] + } + }] + }, + { + "name": "instructorClone", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + }] + }] + }, + { + "name": "instructors", + "type": ["null", "Instructor"] + } + ] +} diff --git a/ballerina/tests/resources/schema_record_nested.json b/ballerina/tests/resources/schema_record_nested.json new file mode 100644 index 0000000..3387664 --- /dev/null +++ b/ballerina/tests/resources/schema_record_nested.json @@ -0,0 +1,49 @@ +{ + "namespace": "example.avro", + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": { + "type": "map", + "values" : "int", + "default": {} + } + }, + { + "name": "age", + "type": "long" + }, + { + "name": "instructor", + "type": { + "name": "Instructor", + "type": "record", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "student", + "type": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + } + } + ] + } + } + ] +} diff --git a/ballerina/tests/resources/schema_record_optional_fields.json b/ballerina/tests/resources/schema_record_optional_fields.json new file mode 100644 index 0000000..847fbb5 --- /dev/null +++ b/ballerina/tests/resources/schema_record_optional_fields.json @@ -0,0 +1,54 @@ +{ + "type": "record", + "name": "Lecturer5", + "fields": [ + { + "name": "name", + "type": [ + "null", + { + "type": "map", + "values": "int" + } + ] + }, + { + "name": "bytes", + "type": ["null", "bytes"] + }, + { + "name": "instructorClone", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }] + }, + { + "name": "instructors", + "type": ["null", "Instructor"] + } + ] +} diff --git a/ballerina/tests/resources/schema_record_string_record_union.json b/ballerina/tests/resources/schema_record_string_record_union.json new file mode 100644 index 0000000..5c3a3d6 --- /dev/null +++ b/ballerina/tests/resources/schema_record_string_record_union.json @@ -0,0 +1,40 @@ +{ + "type": "record", + "name": "Course", + "namespace": "example.avro", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "value", + "type": "float" + }, + { + "name": "credits", + "type": ["null", "int"] + }, + { + "name": "student", + "type": [ + "null", + { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + }, + "string" + ] + } + ] +} diff --git a/ballerina/tests/resources/schema_record_union.json b/ballerina/tests/resources/schema_record_union.json new file mode 100644 index 0000000..2fa5b0c --- /dev/null +++ b/ballerina/tests/resources/schema_record_union.json @@ -0,0 +1,36 @@ +{ + "type": "record", + "name": "Course", + "namespace": "example.avro", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "value", + "type": "float" + }, + { + "name": "credits", + "type": ["null", "int"] + }, + { + "name": "student", + "type": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + } + ] +} diff --git a/ballerina/tests/test.bal b/ballerina/tests/test.bal index f377e16..6309bd6 100644 --- a/ballerina/tests/test.bal +++ b/ballerina/tests/test.bal @@ -17,21 +17,13 @@ import ballerina/io; import ballerina/test; -public type UnionEnumRecord record { - string|Numbers? field1; -}; - -public type UnionFixedRecord record { - string|byte[]? field1; -}; - -public type UnionRec record { - string|UnionEnumRecord? field1; -}; - -public type ReadOnlyRec readonly & record { - string|UnionEnumRecord? & readonly field1; -}; +public isolated function verifyOperation(typedesc providedType, + anydata value, string schema) returns error? { + Schema avro = check new (schema); + byte[] serializedValue = check avro.toAvro(value); + var deserializedValue = check avro.fromAvro(serializedValue, providedType); + test:assertEquals(deserializedValue, value); +} @test:Config { groups: ["enum", "union"] @@ -43,10 +35,7 @@ public isolated function testUnionEnums() returns error? { UnionEnumRecord number = { field1: ONE }; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(number); - UnionEnumRecord deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(number, deserializedValue); + return verifyOperation(UnionEnumRecord, number, schema); } @test:Config { @@ -59,10 +48,7 @@ public isolated function testUnionFixed() returns error? { UnionFixedRecord number = { field1: "ON".toBytes() }; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(number); - UnionFixedRecord deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(number, deserializedValue); + return verifyOperation(UnionFixedRecord, number, schema); } @test:Config { @@ -72,13 +58,10 @@ public isolated function testUnionFixeWithReadOnlyValues() returns error? { string jsonFileName = string `tests/resources/schema_union_fixed_strings.json`; string schema = (check io:fileReadJson(jsonFileName)).toString(); - UnionFixedRecord & readonly number = { + ReadOnlyUnionFixed number = { field1: "ON".toBytes().cloneReadOnly() }; - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(number); - UnionFixedRecord & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(number, deserializedValue); + return verifyOperation(ReadOnlyUnionFixed, number, schema); } @test:Config { @@ -93,11 +76,7 @@ public isolated function testUnionsWithRecordsAndStrings() returns error? { field1: "ONE" } }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(number); - UnionRec deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(number, deserializedValue); + return verifyOperation(UnionRec, number, schema); } @test:Config { @@ -112,11 +91,7 @@ public isolated function testUnionsWithReadOnlyRecords() returns error? { field1: "ONE".cloneReadOnly() } }; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(number); - ReadOnlyRec deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(number, deserializedValue); + return verifyOperation(ReadOnlyRec, number, schema); } @test:Config { @@ -132,11 +107,7 @@ public isolated function testEnums() returns error? { }`; Numbers number = "ONE"; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(number); - Numbers deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(number, deserializedValue); + return verifyOperation(Numbers, number, schema); } @test:Config { @@ -152,11 +123,7 @@ public isolated function testEnumsWithReadOnlyValues() returns error? { }`; Numbers & readonly number = "ONE"; - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(number); - Numbers & readonly deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(number, deserializedValue); + return verifyOperation(Numbers, number, schema); } @test:Config { @@ -208,11 +175,7 @@ public isolated function testFixed() returns error? { }`; byte[] value = "u00ffffffffffffx".toBytes(); - - Schema avro = check new (schema); - byte[] serializedValue = check avro.toAvro(value); - byte[] deserializedValue = check avro.fromAvro(serializedValue); - test:assertEquals(deserializedValue, value); + return verifyOperation(ByteArray, value, schema); } @test:Config { @@ -237,12 +200,7 @@ public function testDbSchemaWithRecords() returns error? { SchemaChangeKey changeKey = { databaseName: "my-db" }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(changeKey); - SchemaChangeKey deserializedValue = check avro.fromAvro(serialize); - test:assertEquals(changeKey, deserializedValue); - + return verifyOperation(SchemaChangeKey, changeKey, schema); } @test:Config { @@ -313,11 +271,7 @@ public function testComplexDbSchema() returns error? { data_collection_order: 1 } }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(envelope); - Envelope deserializedValue = check avro.fromAvro(serialize); - test:assertEquals(deserializedValue, envelope); + return verifyOperation(Envelope, envelope, schema); } @test:Config { @@ -438,9 +392,5 @@ public function testComplexDbSchemaWithNestedRecords() returns error? { }, MessageSource: "MessageSource" }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(envelope2); - Envelope2 deserializedValue = check avro.fromAvro(serialize); - test:assertEquals(deserializedValue, envelope2); + return verifyOperation(Envelope2, envelope2, schema); } diff --git a/ballerina/tests/types.bal b/ballerina/tests/types.bal index 15ce307..0c613c9 100644 --- a/ballerina/tests/types.bal +++ b/ballerina/tests/types.bal @@ -270,3 +270,67 @@ public type Envelope2 record { Block2? 'transaction; string? MessageSource; }; + + +public type UnionEnumRecord record { + string|Numbers? field1; +}; + +public type UnionFixedRecord record { + string|byte[]? field1; +}; + +public type UnionRec record { + string|UnionEnumRecord? field1; +}; + +public type ReadOnlyRec readonly & record { + string|UnionEnumRecord? & readonly field1; +}; + +type ReadOnlyUnionFixed UnionFixedRecord & readonly; +type ByteArray byte[]; +type ReadOnlyIntArray int[] & readonly; +type ReadOnlyStringArray string[] & readonly; +type StringArray string[]; +type String2DArray string[][]; +type ReadOnlyColors Color & readonly; +type NullType (); +type ByteArrayMap map; +type MapOfByteArrayMap map>; +type ReadOnlyMapOfReadOnlyRecord map & readonly; +type ReadOnlyMapOfRecord map & readonly; +type RecordMap map; +type IntMap map; +type EnumMap map; +type EnumArrayMap map; +type FloatMap map; +type FloatArrayMap map; +type StringMap map; +type BooleanMap map; +type ReadOnlyBooleanMap map & readonly; +type MapOfIntMap map>; +type ReadOnlyMapOfReadOnlyMap map>> & readonly; +type MapOfMap map>>; +type IntArrayMap map; +type ReadOnlyIntArrayMap map & readonly; +type StringArrayMap map; +type ByteArrayArrayMap map; +type BooleanArrayMap map; +type MapOfRecordArray map; +type MapOfArrayOfRecordArray map; +type StringArrayArrayMap map; +type MapOfRecordMap map>; +type MapOfRecordArrayMap map>; +type ReadOnlyMapOfRecordArray map> & readonly; +type ArrayOfByteArray byte[][]; +type ReadOnlyStudent2DArray Student[][] & readonly; +type Student2DArray Student[][]; +type ReadOnlyStudentArray Student[] & readonly; +type StudentArray Student[]; +type BooleanArray boolean[]; +type IntArray int[]; +type FloatArray float[]; +type EnumArray Numbers[]; +type Enum2DArray Numbers[][]; +type ReadOnlyString2DArray string[][] & readonly; From b81fea914da571da79f3a4a6e9c9ba7c475d206d Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 13 May 2024 12:29:01 +0530 Subject: [PATCH 44/47] Fix visitor APIs in Serializer classes --- .../lib/avro/serialize/MessageFactory.java | 2 +- ...rializer.java => PrimitiveSerializer.java} | 6 +-- .../lib/avro/serialize/StringSerializer.java | 34 ------------- .../lib/avro/serialize/UnionSerializer.java | 2 +- .../serialize/visitor/ISerializeVisitor.java | 5 +- .../serialize/visitor/SerializeVisitor.java | 50 ++++++++++++------- 6 files changed, 38 insertions(+), 61 deletions(-) rename native/src/main/java/io/ballerina/lib/avro/serialize/{PrimitiveDeserializer.java => PrimitiveSerializer.java} (87%) delete mode 100644 native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java index 596eb26..1d29b11 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java @@ -32,7 +32,7 @@ public static Serializer createMessage(Schema schema) { case MAP -> new MapSerializer(schema); case RECORD -> new RecordSerializer(schema); case BYTES -> new ByteSerializer(); - default -> new PrimitiveDeserializer(schema); + default -> new PrimitiveSerializer(schema); }; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveSerializer.java similarity index 87% rename from native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveDeserializer.java rename to native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveSerializer.java index f6bc65a..080c1d4 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveSerializer.java @@ -21,14 +21,14 @@ import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import org.apache.avro.Schema; -public class PrimitiveDeserializer extends Serializer { +public class PrimitiveSerializer extends Serializer { - public PrimitiveDeserializer(Schema schema) { + public PrimitiveSerializer(Schema schema) { super(schema); } @Override - public Object convert(SerializeVisitor serializeVisitor, Object data) { + public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { return serializeVisitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java deleted file mode 100644 index ade3fb6..0000000 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/StringSerializer.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.ballerina.lib.avro.serialize; - -import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; -import org.apache.avro.Schema; - -public class StringSerializer extends Serializer { - - public StringSerializer(Schema schema) { - super(schema); - } - - @Override - public Object convert(SerializeVisitor serializeVisitor, Object data) { - return serializeVisitor.visitString(data); - } -} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java index 0807a7b..a4ef7f3 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java @@ -29,6 +29,6 @@ public UnionSerializer(Schema schema) { @Override public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { - return serializeVisitor.visitUnion(this, data); + return serializeVisitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java index 0290c99..ffe79de 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java @@ -21,7 +21,7 @@ import io.ballerina.lib.avro.serialize.ArraySerializer; import io.ballerina.lib.avro.serialize.EnumSerializer; import io.ballerina.lib.avro.serialize.FixedSerializer; -import io.ballerina.lib.avro.serialize.PrimitiveDeserializer; +import io.ballerina.lib.avro.serialize.PrimitiveSerializer; import io.ballerina.lib.avro.serialize.RecordSerializer; import io.ballerina.runtime.api.values.BArray; import io.ballerina.runtime.api.values.BMap; @@ -30,10 +30,9 @@ public interface ISerializeVisitor { - String visitString(Object data); GenericRecord visit(RecordSerializer recordSerializer, BMap data) throws Exception; GenericData.Array visit(ArraySerializer arraySerializer, BArray data); Object visit(EnumSerializer enumSerializer, Object data); GenericData.Fixed visit(FixedSerializer fixedSerializer, Object data); - Object visit(PrimitiveDeserializer primitiveDeserializer, Object data); + Object visit(PrimitiveSerializer primitiveSerializer, Object data) throws Exception; } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java index 2026e71..4c52977 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java @@ -22,7 +22,7 @@ import io.ballerina.lib.avro.serialize.EnumSerializer; import io.ballerina.lib.avro.serialize.FixedSerializer; import io.ballerina.lib.avro.serialize.MapSerializer; -import io.ballerina.lib.avro.serialize.PrimitiveDeserializer; +import io.ballerina.lib.avro.serialize.PrimitiveSerializer; import io.ballerina.lib.avro.serialize.RecordSerializer; import io.ballerina.lib.avro.serialize.Serializer; import io.ballerina.lib.avro.serialize.UnionSerializer; @@ -34,7 +34,6 @@ import io.ballerina.runtime.api.utils.TypeUtils; import io.ballerina.runtime.api.values.BArray; import io.ballerina.runtime.api.values.BMap; -import io.ballerina.runtime.api.values.BString; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; @@ -50,7 +49,7 @@ public class SerializeVisitor implements ISerializeVisitor { public Serializer createSerializer(Schema schema) { return switch (schema.getValueType().getType()) { case INT, LONG, FLOAT, DOUBLE, BOOLEAN, STRING, BYTES -> - new PrimitiveDeserializer(schema.getValueType()); + new PrimitiveSerializer(schema.getValueType()); case RECORD -> new RecordSerializer(schema.getValueType()); case MAP -> @@ -66,11 +65,6 @@ public Serializer createSerializer(Schema schema) { }; } - @Override - public String visitString(Object data) { - return ((BString) data).getValue(); - } - @Override public GenericRecord visit(RecordSerializer recordSerializer, BMap data) throws Exception { GenericRecord genericRecord = new GenericData.Record(recordSerializer.getSchema()); @@ -95,13 +89,13 @@ private Object serializeField(Schema schema, Object fieldData) throws Exception case UNION -> new UnionSerializer(schema).convert(this, fieldData); default -> - new PrimitiveDeserializer(schema).convert(this, fieldData); + new PrimitiveSerializer(schema).convert(this, fieldData); }; } @Override - public Object visit(PrimitiveDeserializer primitiveDeserializer, Object data) { - switch (primitiveDeserializer.getSchema().getType()) { + public Object visit(PrimitiveSerializer primitiveSerializer, Object data) throws Exception { + switch (primitiveSerializer.getSchema().getType()) { case INT -> { return ((Long) data).intValue(); } @@ -117,6 +111,12 @@ public Object visit(PrimitiveDeserializer primitiveDeserializer, Object data) { case STRING -> { return data.toString(); } + case NULL -> { + if (data != null) { + throw new Exception("The value does not match with the null schema"); + } + return null; + } default -> { return data; } @@ -156,12 +156,12 @@ public GenericData.Array visit(ArraySerializer arraySerializer, BArray d return Objects.requireNonNull(visitor).visit(data, arraySerializer.getSchema(), array); } - public Object visitUnion(UnionSerializer unionSerializer, Object data) throws Exception { + public Object visit(UnionSerializer unionSerializer, Object data) throws Exception { Schema fieldSchema = unionSerializer.getSchema(); Type typeName = TypeUtils.getType(data); switch (typeName.getTag()) { case TypeTags.STRING_TAG -> { - return visitiUnionStrings(data, fieldSchema); + return visitUnionStrings(data, fieldSchema); } case TypeTags.ARRAY_TAG -> { return visitUnionArrays(data, fieldSchema); @@ -189,7 +189,13 @@ private Object visitUnionFloats(Object data, Schema fieldSchema) { return fieldSchema.getTypes().stream() .filter(schema -> schema.getType().equals(Schema.Type.FLOAT)) .findFirst() - .map(schema -> new PrimitiveDeserializer(schema).convert(this, data)) + .map(schema -> { + try { + return new PrimitiveSerializer(schema).convert(this, data); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) .orElse(data); } @@ -197,23 +203,29 @@ private Object visitUnionIntegers(Object data, Schema fieldSchema) { return fieldSchema.getTypes().stream() .filter(schema -> schema.getType().equals(Schema.Type.INT)) .findFirst() - .map(schema -> new PrimitiveDeserializer(schema).convert(this, data)) + .map(schema -> { + try { + return new PrimitiveSerializer(schema).convert(this, data); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) .orElse(data); } - private Object visitiUnionStrings(Object data, Schema fieldSchema) { + private Object visitUnionStrings(Object data, Schema fieldSchema) throws Exception { return fieldSchema.getTypes().stream() .filter(type -> type.getType().equals(Schema.Type.ENUM)) .findFirst() .map(type -> visit(new EnumSerializer(type), data)) - .orElse(visit(new PrimitiveDeserializer(fieldSchema), data.toString())); + .orElse(visit(new PrimitiveSerializer(fieldSchema), data.toString())); } - private Object visitUnionArrays(Object data, Schema fieldSchema) { + private Object visitUnionArrays(Object data, Schema fieldSchema) throws Exception { for (Schema schema : fieldSchema.getTypes()) { switch (schema.getType()) { case BYTES -> { - return new PrimitiveDeserializer(schema).convert(this, data); + return new PrimitiveSerializer(schema).convert(this, data); } case FIXED -> { return new FixedSerializer(schema).convert(this, data); From 8468be6e8d91851977785f12ac499d76bd853d61 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 13 May 2024 12:30:07 +0530 Subject: [PATCH 45/47] Fix APIs in Deserializer classes --- .../main/java/io/ballerina/lib/avro/Avro.java | 2 +- .../avro/deserialize/ArrayDeserializer.java | 4 ++-- .../avro/deserialize/ByteDeserializer.java | 11 ++++++---- .../lib/avro/deserialize/Deserializer.java | 4 ++-- .../avro/deserialize/DoubleDeserializer.java | 4 ++-- .../avro/deserialize/EnumDeserializer.java | 4 ++-- .../avro/deserialize/FixedDeserializer.java | 4 ++-- .../lib/avro/deserialize/MapDeserializer.java | 4 ++-- .../avro/deserialize/NullDeserializer.java | 4 ++-- .../deserialize/PrimitiveDeserializer.java | 4 ++-- .../avro/deserialize/RecordDeserializer.java | 4 ++-- .../avro/deserialize/StringDeserializer.java | 4 ++-- .../avro/deserialize/UnionDeserializer.java | 4 ++-- .../visitor/DeserializeArrayVisitor.java | 2 +- .../visitor/DeserializeRecordVisitor.java | 8 +++---- .../visitor/DeserializeVisitor.java | 22 ++++++++----------- .../avro/deserialize/visitor/RecordUtils.java | 8 +++---- .../deserialize/visitor/UnionRecordUtils.java | 2 +- .../lib/avro/serialize/MessageFactory.java | 2 -- 19 files changed, 49 insertions(+), 52 deletions(-) diff --git a/native/src/main/java/io/ballerina/lib/avro/Avro.java b/native/src/main/java/io/ballerina/lib/avro/Avro.java index 41e06fa..fc65662 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Avro.java +++ b/native/src/main/java/io/ballerina/lib/avro/Avro.java @@ -81,7 +81,7 @@ public static Object fromAvro(BObject schemaObject, BArray payload, BTypedesc ty Object data = datumReader.read(payload, decoder); DeserializeVisitor deserializeVisitor = new DeserializeVisitor(); Deserializer deserializer = DeserializeFactory.generateDeserializer(schema, typeParam.getDescribingType()); - return Objects.requireNonNull(deserializer).visit(deserializeVisitor, data); + return Objects.requireNonNull(deserializer).accept(deserializeVisitor, data); } catch (Exception e) { return createError(DESERIALIZATION_ERROR, e); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java index 5fc0c18..417599e 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java @@ -30,12 +30,12 @@ public ArrayDeserializer(Type type, Schema schema) { } @Override - public Object visit(DeserializeVisitor visitor, Object data) throws Exception { + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visit(this, (GenericData.Array) data); } @Override - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java index 0c51188..4e228d7 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java @@ -19,17 +19,20 @@ package io.ballerina.lib.avro.deserialize; import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.creators.ValueCreator; import org.apache.avro.generic.GenericData; +import java.nio.ByteBuffer; + public class ByteDeserializer extends Deserializer { @Override - public Object visit(DeserializeVisitor visitor, Object data) throws Exception { - return visitor.visitBytes(data); + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { + return ValueCreator.createArrayValue(((ByteBuffer) data).array()); } @Override - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { - return visitor.visitBytes(data); + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return null; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java index d7897a7..85d8930 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java @@ -46,6 +46,6 @@ public Type getType() { return TypeUtils.getReferredType(type); } - public abstract Object visit(DeserializeVisitor visitor, Object data) throws Exception; - public abstract Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception; + public abstract Object accept(DeserializeVisitor visitor, Object data) throws Exception; + public abstract Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception; } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java index d9f9c86..4529dce 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java @@ -24,12 +24,12 @@ public class DoubleDeserializer extends Deserializer { @Override - public Object visit(DeserializeVisitor visitor, Object data) { + public Object accept(DeserializeVisitor visitor, Object data) { return visitor.visitDouble(data); } @Override - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visitDouble(data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java index 45818fe..84b4d6d 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java @@ -30,12 +30,12 @@ public EnumDeserializer(Type type, Schema schema) { } @Override - public Object visit(DeserializeVisitor visitor, Object data) { + public Object accept(DeserializeVisitor visitor, Object data) { return visitor.visit(this, (GenericData.Array) data); } @Override - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java index f823d4b..03cf365 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java @@ -30,12 +30,12 @@ public FixedDeserializer(Type type, Schema schema) { } @Override - public Object visit(DeserializeVisitor visitor, Object data) { + public Object accept(DeserializeVisitor visitor, Object data) { return visitor.visit(this, data); } @Override - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java index 0f01ea4..a3c8623 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java @@ -32,11 +32,11 @@ public MapDeserializer(Schema schema, Type type) { } @Override - public Object visit(DeserializeVisitor visitor, Object data) throws Exception { + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visit(this, (Map) data); } - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return null; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java index 7218a12..be67f72 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java @@ -24,14 +24,14 @@ public class NullDeserializer extends Deserializer { @Override - public Object visit(DeserializeVisitor visitor, Object data) throws Exception { + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { if (data != null) { throw new Exception("The value does not match with the null schema"); } return null; } - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return null; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java index a865b45..5f3ed99 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java @@ -30,12 +30,12 @@ public PrimitiveDeserializer(Type type, Schema schema) { } @Override - public Object visit(DeserializeVisitor visitor, Object data) { + public Object accept(DeserializeVisitor visitor, Object data) { return visitor.visit(this, data); } @Override - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java index e27daa1..c29828d 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java @@ -31,11 +31,11 @@ public RecordDeserializer(Type type, Schema schema) { } @Override - public Object visit(DeserializeVisitor visitor, Object data) throws Exception { + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visit(this, (GenericRecord) data); } - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java index 303613c..bd47d2c 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java @@ -24,11 +24,11 @@ public class StringDeserializer extends Deserializer { @Override - public Object visit(DeserializeVisitor visitor, Object data) { + public Object accept(DeserializeVisitor visitor, Object data) { return visitor.visitString(data); } - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return null; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java index 3a6ea6f..f4112e6 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java @@ -30,11 +30,11 @@ public UnionDeserializer(Type type, Schema schema) { } @Override - public Object visit(DeserializeVisitor visitor, Object data) throws Exception { + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visit(this, (GenericData.Array) data); } - public Object visit(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { return visitor.visit(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java index 455802e..5bb11c2 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java @@ -53,6 +53,6 @@ public Object visit(ArrayDeserializer arrayDeserializer, GenericData.Array data) throws Exception { Deserializer deserializer = createDeserializer(arrayDeserializer.getSchema(), arrayDeserializer.getType()); - return deserializer.visit(this, data); + return deserializer.accept(this, data); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java index be711e8..0931946 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java @@ -78,14 +78,14 @@ private void processMapField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { Type mapType = extractMapType(avroRecord.getType()); MapDeserializer mapDeserializer = new MapDeserializer(field.schema(), mapType); - Object fieldValue = mapDeserializer.visit(this, fieldData); + Object fieldValue = mapDeserializer.accept(this, fieldData); avroRecord.put(StringUtils.fromString(field.name()), fieldValue); } private void processArrayField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { ArrayDeserializer arrayDes = new ArrayDeserializer(avroRecord.getType(), field.schema()); - Object fieldValue = arrayDes.visit(this, (GenericData.Array) fieldData); + Object fieldValue = arrayDes.accept(this, (GenericData.Array) fieldData); avroRecord.put(StringUtils.fromString(field.name()), fieldValue); } @@ -99,14 +99,14 @@ private void processRecordField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { Type recType = extractRecordType((RecordType) avroRecord.getType()); RecordDeserializer recordDes = new RecordDeserializer(recType, field.schema()); - Object fieldValue = recordDes.visit(this, (GenericRecord) fieldData); + Object fieldValue = recordDes.accept(this, (GenericRecord) fieldData); avroRecord.put(StringUtils.fromString(field.name()), fieldValue); } private void processStringField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { StringDeserializer stringDes = new StringDeserializer(); - Object fieldValue = stringDes.visit(this, fieldData); + Object fieldValue = stringDes.accept(this, fieldData); avroRecord.put(StringUtils.fromString(field.name()), fieldValue); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java index 419a857..2c8be4c 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -206,14 +206,14 @@ public BArray visit(UnionDeserializer unionDeserializer, GenericData.Array { - return visitBytes(data); + return (BArray) data; } } } private BArray visitRecordArray(GenericData.Array data, Type type, Schema schema) throws Exception { RecordDeserializer recordDeserializer = new RecordDeserializer(type, schema.getElementType()); - return (BArray) recordDeserializer.visit(this, data); + return (BArray) recordDeserializer.accept(this, data); } private BArray visitUnionArray(GenericData.Array data, ArrayType type, Schema schema) throws Exception { @@ -222,7 +222,7 @@ private BArray visitUnionArray(GenericData.Array data, ArrayType type, S ArrayDeserializer arrayDeserializer = new ArrayDeserializer(elementType, schema.getElementType()); int index = 0; for (Object currentData : data) { - Object deserializedObject = arrayDeserializer.visit(this, (GenericData.Array) currentData); + Object deserializedObject = arrayDeserializer.accept(this, (GenericData.Array) currentData); objects[index++] = deserializedObject; } return ValueCreator.createArrayValue(objects, type); @@ -238,14 +238,14 @@ public BArray visit(RecordDeserializer recordDeserializer, GenericData.Array { for (Object datum : data) { Type fieldType = ((ReferenceType) type).getReferredType(); RecordDeserializer recordDes = new RecordDeserializer(fieldType, schema.getElementType()); - recordList.add(recordDes.visit(this, (GenericRecord) datum)); + recordList.add(recordDes.accept(this, (GenericRecord) datum)); } } } @@ -266,7 +266,7 @@ private void processMaps(BMap avroRecord, Schema schema, Schema fieldSchema = schema.getValueType(); Type fieldType = type.getConstrainedType(); MapDeserializer mapDes = new MapDeserializer(fieldSchema, fieldType); - Object fieldValue = mapDes.visit(this, value); + Object fieldValue = mapDes.accept(this, value); avroRecord.put(fromString(key.toString()), fieldValue); } @@ -274,7 +274,7 @@ private void processMapRecord(BMap avroRecord, Schema schema, MapType type, Object key, GenericRecord value) throws Exception { Type fieldType = type.getConstrainedType(); RecordDeserializer recordDes = new RecordDeserializer(fieldType, schema.getValueType()); - Object fieldValue = recordDes.visit(this, value); + Object fieldValue = recordDes.accept(this, value); avroRecord.put(fromString(key.toString()), fieldValue); } @@ -288,7 +288,7 @@ private void processMapArray(BMap avroRecord, Schema schema, public Object visit(ArrayDeserializer arrayDeserializer, GenericData.Array data) throws Exception { Deserializer deserializer = createDeserializer(arrayDeserializer.getSchema(), arrayDeserializer.getType()); - return deserializer.visit(new DeserializeArrayVisitor(), data); + return deserializer.accept(new DeserializeArrayVisitor(), data); } public BArray visit(EnumDeserializer enumDeserializer, GenericData.Array data) { @@ -334,7 +334,7 @@ private static BArray visitIntegerArray(GenericData.Array data, Schema s private BArray visitBytesArray(GenericData.Array data, Type type) { List values = new ArrayList<>(); for (Object datum : data) { - values.add(visitBytes(datum)); + values.add(ValueCreator.createArrayValue(((ByteBuffer) datum).array())); } return ValueCreator.createArrayValue(values.toArray(new BArray[data.size()]), (ArrayType) type); } @@ -391,10 +391,6 @@ public double visitDouble(Object data) { return (double) data; } - public BArray visitBytes(Object data) { - return ValueCreator.createArrayValue(((ByteBuffer) data).array()); - } - public BArray visitFixed(Object data) { GenericData.Fixed fixed = (GenericData.Fixed) data; return ValueCreator.createArrayValue(fixed.bytes()); diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java index 2e031a8..e37e2c6 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java @@ -43,14 +43,14 @@ public static void processMapField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { Type mapType = extractMapType(avroRecord.getType()); MapDeserializer mapDeserializer = new MapDeserializer(field.schema(), mapType); - Object fieldValue = mapDeserializer.visit(new DeserializeVisitor(), fieldData); + Object fieldValue = mapDeserializer.accept(new DeserializeVisitor(), fieldData); avroRecord.put(fromString(field.name()), fieldValue); } public static void processArrayField(BMap avroRecord, Schema.Field field, Object fieldData, Type type) throws Exception { ArrayDeserializer arrayDes = new ArrayDeserializer(type, field.schema()); - Object fieldValue = arrayDes.visit(new DeserializeVisitor(), (GenericData.Array) fieldData); + Object fieldValue = arrayDes.accept(new DeserializeVisitor(), (GenericData.Array) fieldData); avroRecord.put(fromString(field.name()), fieldValue); } @@ -64,14 +64,14 @@ public static void processRecordField(BMap avroRecord, Schema.Field field, Object fieldData) throws Exception { Type recType = extractRecordType((RecordType) avroRecord.getType()); RecordDeserializer recordDes = new RecordDeserializer(recType, field.schema()); - Object fieldValue = recordDes.visit(new DeserializeVisitor(), fieldData); + Object fieldValue = recordDes.accept(new DeserializeVisitor(), fieldData); avroRecord.put(fromString(field.name()), fieldValue); } public static void processStringField(BMap avroRecord, Schema.Field field, Object fieldData) { StringDeserializer stringDes = new StringDeserializer(); - Object fieldValue = stringDes.visit(new DeserializeVisitor(), fieldData); + Object fieldValue = stringDes.accept(new DeserializeVisitor(), fieldData); avroRecord.put(fromString(field.name()), fieldValue); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java index b1f1488..b56484f 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java @@ -116,7 +116,7 @@ public static void handleRecordField(Type type, Schema.Field field, Object field BMap ballerinaRecord, Schema schemaType) throws Exception { if (fieldData instanceof GenericRecord) { RecordDeserializer recordDes = new RecordDeserializer(type, schemaType); - Object fieldValue = recordDes.visit(new DeserializeVisitor(), (GenericRecord) fieldData); + Object fieldValue = recordDes.accept(new DeserializeVisitor(), (GenericRecord) fieldData); ballerinaRecord.put(StringUtils.fromString(field.name()), fieldValue); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java index 1d29b11..ee5bc68 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java @@ -24,8 +24,6 @@ public class MessageFactory { public static Serializer createMessage(Schema schema) { return switch (schema.getType()) { - case NULL -> new NullSerializer(); - case STRING -> new StringSerializer(schema); case ARRAY -> new ArraySerializer(schema); case FIXED -> new FixedSerializer(schema); case ENUM -> new EnumSerializer(schema); From 6197f36395f5a8bba26a54890ebcfa7233afcafc Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 13 May 2024 13:26:02 +0530 Subject: [PATCH 46/47] Combine deserializing all primtive data types --- .../avro/deserialize/ByteDeserializer.java | 38 ------ .../avro/deserialize/DeserializeFactory.java | 4 - .../avro/deserialize/DoubleDeserializer.java | 35 ------ .../avro/deserialize/NullDeserializer.java | 37 ------ .../deserialize/PrimitiveDeserializer.java | 2 +- .../avro/deserialize/StringDeserializer.java | 34 ----- .../visitor/DeserializeRecordVisitor.java | 117 ------------------ .../visitor/DeserializeVisitor.java | 69 +++++++---- .../avro/deserialize/visitor/RecordUtils.java | 6 +- 9 files changed, 51 insertions(+), 291 deletions(-) delete mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java delete mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java delete mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java delete mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java delete mode 100644 native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java deleted file mode 100644 index 4e228d7..0000000 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/ByteDeserializer.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.ballerina.lib.avro.deserialize; - -import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; -import io.ballerina.runtime.api.creators.ValueCreator; -import org.apache.avro.generic.GenericData; - -import java.nio.ByteBuffer; - -public class ByteDeserializer extends Deserializer { - - @Override - public Object accept(DeserializeVisitor visitor, Object data) throws Exception { - return ValueCreator.createArrayValue(((ByteBuffer) data).array()); - } - - @Override - public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { - return null; - } -} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java index 3be1944..42069a4 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java @@ -25,14 +25,10 @@ public class DeserializeFactory { public static Deserializer generateDeserializer(Schema schema, Type type) { return switch (schema.getType()) { - case NULL -> new NullDeserializer(); - case FLOAT, DOUBLE -> new DoubleDeserializer(); - case STRING, ENUM -> new StringDeserializer(); case ARRAY -> new ArrayDeserializer(type, schema); case FIXED -> new FixedDeserializer(type, schema); case MAP -> new MapDeserializer(schema, type); case RECORD -> new RecordDeserializer(type, schema); - case BYTES -> new ByteDeserializer(); default -> new PrimitiveDeserializer(type, schema); }; } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java deleted file mode 100644 index 4529dce..0000000 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/DoubleDeserializer.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.ballerina.lib.avro.deserialize; - -import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; -import org.apache.avro.generic.GenericData; - -public class DoubleDeserializer extends Deserializer { - - @Override - public Object accept(DeserializeVisitor visitor, Object data) { - return visitor.visitDouble(data); - } - - @Override - public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { - return visitor.visitDouble(data); - } -} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java deleted file mode 100644 index be67f72..0000000 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/NullDeserializer.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.ballerina.lib.avro.deserialize; - -import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; -import org.apache.avro.generic.GenericData; - -public class NullDeserializer extends Deserializer { - - @Override - public Object accept(DeserializeVisitor visitor, Object data) throws Exception { - if (data != null) { - throw new Exception("The value does not match with the null schema"); - } - return null; - } - - public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { - return null; - } -} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java index 5f3ed99..ae2600f 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java @@ -30,7 +30,7 @@ public PrimitiveDeserializer(Type type, Schema schema) { } @Override - public Object accept(DeserializeVisitor visitor, Object data) { + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { return visitor.visit(this, data); } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java deleted file mode 100644 index bd47d2c..0000000 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/StringDeserializer.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.ballerina.lib.avro.deserialize; - -import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; -import org.apache.avro.generic.GenericData; - -public class StringDeserializer extends Deserializer { - - @Override - public Object accept(DeserializeVisitor visitor, Object data) { - return visitor.visitString(data); - } - - public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { - return null; - } -} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java deleted file mode 100644 index 0931946..0000000 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeRecordVisitor.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) - * - * WSO2 LLC. licenses this file to you under the Apache License, - * Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package io.ballerina.lib.avro.deserialize.visitor; - -import io.ballerina.lib.avro.deserialize.ArrayDeserializer; -import io.ballerina.lib.avro.deserialize.MapDeserializer; -import io.ballerina.lib.avro.deserialize.RecordDeserializer; -import io.ballerina.lib.avro.deserialize.StringDeserializer; -import io.ballerina.runtime.api.creators.ValueCreator; -import io.ballerina.runtime.api.types.RecordType; -import io.ballerina.runtime.api.types.Type; -import io.ballerina.runtime.api.utils.StringUtils; -import io.ballerina.runtime.api.values.BMap; -import io.ballerina.runtime.api.values.BString; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericRecord; - -import java.nio.ByteBuffer; - -import static io.ballerina.lib.avro.Utils.getMutableType; -import static io.ballerina.lib.avro.deserialize.visitor.UnionRecordUtils.visitUnionRecords; - -public class DeserializeRecordVisitor extends DeserializeVisitor { - - public BMap visit(RecordDeserializer recordDeserializer, GenericRecord rec) throws Exception { - Type originalType = recordDeserializer.getType(); - Type type = recordDeserializer.getType(); - Schema schema = recordDeserializer.getSchema(); - BMap avroRecord = ValueCreator.createRecordValue((RecordType) getMutableType(type)); - for (Schema.Field field : schema.getFields()) { - Object fieldData = rec.get(field.name()); - switch (field.schema().getType()) { - case MAP -> - processMapField(avroRecord, field, fieldData); - case ARRAY -> - processArrayField(avroRecord, field, fieldData); - case BYTES -> - processBytesField(avroRecord, field, fieldData); - case RECORD -> - processRecordField(avroRecord, field, fieldData); - case STRING -> - processStringField(avroRecord, field, fieldData); - case INT -> - avroRecord.put(StringUtils.fromString(field.name()), Long.parseLong(fieldData.toString())); - case FLOAT -> - avroRecord.put(StringUtils.fromString(field.name()), Double.parseDouble(fieldData.toString())); - case UNION -> - processUnionField(type, avroRecord, field, fieldData); - default -> - avroRecord.put(StringUtils.fromString(field.name()), fieldData); - } - } - - if (originalType.isReadOnly()) { - avroRecord.freezeDirect(); - } - return avroRecord; - } - - private void processMapField(BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - Type mapType = extractMapType(avroRecord.getType()); - MapDeserializer mapDeserializer = new MapDeserializer(field.schema(), mapType); - Object fieldValue = mapDeserializer.accept(this, fieldData); - avroRecord.put(StringUtils.fromString(field.name()), fieldValue); - } - - private void processArrayField(BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - ArrayDeserializer arrayDes = new ArrayDeserializer(avroRecord.getType(), field.schema()); - Object fieldValue = arrayDes.accept(this, (GenericData.Array) fieldData); - avroRecord.put(StringUtils.fromString(field.name()), fieldValue); - } - - private void processBytesField(BMap avroRecord, Schema.Field field, Object fieldData) { - ByteBuffer byteBuffer = (ByteBuffer) fieldData; - Object fieldValue = ValueCreator.createArrayValue(byteBuffer.array()); - avroRecord.put(StringUtils.fromString(field.name()), fieldValue); - } - - private void processRecordField(BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - Type recType = extractRecordType((RecordType) avroRecord.getType()); - RecordDeserializer recordDes = new RecordDeserializer(recType, field.schema()); - Object fieldValue = recordDes.accept(this, (GenericRecord) fieldData); - avroRecord.put(StringUtils.fromString(field.name()), fieldValue); - } - - private void processStringField(BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - StringDeserializer stringDes = new StringDeserializer(); - Object fieldValue = stringDes.accept(this, fieldData); - avroRecord.put(StringUtils.fromString(field.name()), fieldValue); - } - - private void processUnionField(Type type, BMap avroRecord, - Schema.Field field, Object fieldData) throws Exception { - visitUnionRecords(type, avroRecord, field, fieldData); - } -} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java index 2c8be4c..b03d8eb 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -153,33 +153,58 @@ public BMap visit(MapDeserializer mapDeserializer, Map) ValueUtils.convert(avroRecord, type); } - public Object visit(PrimitiveDeserializer primitiveDeserializer, Object data) { + public Object visit(PrimitiveDeserializer primitiveDeserializer, Object data) throws Exception { Schema schema = primitiveDeserializer.getSchema(); Type type = primitiveDeserializer.getType(); - if (schema.getType().equals(Schema.Type.ARRAY)) { - GenericData.Array array = (GenericData.Array) data; - switch (schema.getElementType().getType()) { - case STRING -> { - return ValueUtils.convert(visitStringArray(array), type); - } - case INT -> { - return ValueUtils.convert(visitIntArray(array), type); - } - case LONG -> { - return ValueUtils.convert(visitLongArray(array), type); - } - case FLOAT, DOUBLE -> { - return ValueUtils.convert(visitDoubleArray(array), type); - } - case BOOLEAN -> { - return ValueUtils.convert(visitBooleanArray(array), type); + switch(schema.getType()) { + case ARRAY -> { + return visitPrimitiveArrays(primitiveDeserializer, (GenericData.Array) data, schema, type); + } + case STRING, ENUM -> { + return StringUtils.fromString(data.toString()); + } + case FLOAT, DOUBLE -> { + if (data instanceof Float) { + return Double.parseDouble(data.toString()); } - default -> { - return ValueUtils.convert(visitBytesArray(array, primitiveDeserializer.getType()), type); + return data; + } + case NULL -> { + if (data != null) { + throw new Exception("The value does not match with the null schema"); } + return null; + } + case BYTES -> { + return ValueCreator.createArrayValue(((ByteBuffer) data).array()); + } + default -> { + return data; + } + } + } + + private Object visitPrimitiveArrays(PrimitiveDeserializer primitiveDeserializer, GenericData.Array data, + Schema schema, Type type) { + switch (schema.getElementType().getType()) { + case STRING -> { + return ValueUtils.convert(visitStringArray(data), type); + } + case INT -> { + return ValueUtils.convert(visitIntArray(data), type); + } + case LONG -> { + return ValueUtils.convert(visitLongArray(data), type); + } + case FLOAT, DOUBLE -> { + return ValueUtils.convert(visitDoubleArray(data), type); + } + case BOOLEAN -> { + return ValueUtils.convert(visitBooleanArray(data), type); + } + default -> { + return ValueUtils.convert(visitBytesArray(data, primitiveDeserializer.getType()), type); } - } else { - return data; } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java index e37e2c6..42d0411 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java @@ -20,8 +20,8 @@ import io.ballerina.lib.avro.deserialize.ArrayDeserializer; import io.ballerina.lib.avro.deserialize.MapDeserializer; +import io.ballerina.lib.avro.deserialize.PrimitiveDeserializer; import io.ballerina.lib.avro.deserialize.RecordDeserializer; -import io.ballerina.lib.avro.deserialize.StringDeserializer; import io.ballerina.runtime.api.creators.ValueCreator; import io.ballerina.runtime.api.types.RecordType; import io.ballerina.runtime.api.types.Type; @@ -69,8 +69,8 @@ public static void processRecordField(BMap avroRecord, } public static void processStringField(BMap avroRecord, - Schema.Field field, Object fieldData) { - StringDeserializer stringDes = new StringDeserializer(); + Schema.Field field, Object fieldData) throws Exception { + PrimitiveDeserializer stringDes = new PrimitiveDeserializer(null, field.schema()); Object fieldValue = stringDes.accept(new DeserializeVisitor(), fieldData); avroRecord.put(fromString(field.name()), fieldValue); } From 6205b51e1445c8caca5e91d2a700dd403160e471 Mon Sep 17 00:00:00 2001 From: Nuvindu Date: Mon, 13 May 2024 13:27:27 +0530 Subject: [PATCH 47/47] Exclude spotbugs to avoid reinitiating field values --- build-config/spotbugs-exclude.xml | 14 ++++++++++++++ .../lib/avro/deserialize/Deserializer.java | 6 +++--- .../ballerina/lib/avro/serialize/Serializer.java | 10 +++++----- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/build-config/spotbugs-exclude.xml b/build-config/spotbugs-exclude.xml index 2895ed4..68bb7b6 100644 --- a/build-config/spotbugs-exclude.xml +++ b/build-config/spotbugs-exclude.xml @@ -26,7 +26,21 @@ + + + + + + + + + + + + + + diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java index 85d8930..ea48aa4 100644 --- a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java @@ -34,16 +34,16 @@ public Deserializer() { } public Deserializer(Type type, Schema schema) { - this.schema = schema == null ? null : new Schema.Parser().parse(schema.toString()); + this.schema = schema == null ? null : schema; this.type = type == null ? null : TypeUtils.getReferredType(type); } public Schema getSchema() { - return new Schema.Parser().parse(schema.toString()); + return this.schema; } public Type getType() { - return TypeUtils.getReferredType(type); + return this.type; } public abstract Object accept(DeserializeVisitor visitor, Object data) throws Exception; diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java index 67fa6ae..ef0f1f2 100644 --- a/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java @@ -25,7 +25,7 @@ public abstract class Serializer { - private final String schema; + private final Schema schema; private final Type type; public Serializer() { @@ -35,20 +35,20 @@ public Serializer() { public Serializer(Schema schema) { this.type = null; - this.schema = schema.toString(); + this.schema = schema; } public Serializer(Schema schema, Type type) { this.type = TypeUtils.getImpliedType(type); - this.schema = schema.toString(); + this.schema = schema; } public Schema getSchema() { - return new Schema.Parser().parse(schema); + return this.schema; } public Type getType() { - return TypeUtils.getImpliedType(type); + return this.type; } public abstract Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception;