diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 36cac9b..1d79af3 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "avro" -version = "0.1.2" +version = "0.1.3" authors = ["Ballerina"] export=["avro"] keywords = ["avro", "serialization", "deserialization", "serdes"] @@ -18,8 +18,8 @@ graalvmCompatible = true [[platform.java17.dependency]] groupId = "io.ballerina.lib" artifactId = "avro-native" -version = "0.1.2" -path = "../native/build/libs/avro-native-0.1.2.jar" +version = "0.1.3" +path = "../native/build/libs/avro-native-0.1.3-SNAPSHOT.jar" [[platform.java17.dependency]] groupId = "org.apache.avro" @@ -44,9 +44,3 @@ groupId = "com.fasterxml.jackson.core" artifactId = "jackson-databind" version = "2.17.0" path = "./lib/jackson-databind-2.17.0.jar" - -[[platform.java11.dependency]] -groupId = "com.fasterxml.jackson.dataformat" -artifactId = "jackson-dataformat-avro" -version = "2.17.0" -path = "./lib/jackson-dataformat-avro-2.17.0.jar" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 98def43..02a1600 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -10,7 +10,7 @@ distribution-version = "2201.9.0" [[package]] org = "ballerina" name = "avro" -version = "0.1.2" +version = "0.1.3" dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"}, diff --git a/ballerina/build.gradle b/ballerina/build.gradle index 734004c..f0d3337 100644 --- a/ballerina/build.gradle +++ b/ballerina/build.gradle @@ -72,9 +72,6 @@ dependencies { externalJars(group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jacksonVersion}") { transitive = false } - externalJars(group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-avro', version: "${jacksonVersion}") { - transitive = false - } } task updateTomlFiles { diff --git a/ballerina/tests/array_tests.bal b/ballerina/tests/array_tests.bal new file mode 100644 index 0000000..50aa893 --- /dev/null +++ b/ballerina/tests/array_tests.bal @@ -0,0 +1,371 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; +import ballerina/io; + +@test:Config { + groups: ["array", "int"] +} +public isolated function testIntArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "integers", + "namespace": "data", + "items": "int" + }`; + + int[] numbers = [22, 556, 78]; + return verifyOperation(IntArray, numbers, schema); +} + +@test:Config { + groups: ["array", "int", "qqq"] +} +public isolated function testReadOnlyIntArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "integers", + "namespace": "data", + "items": "int" + }`; + + int[] & readonly numbers = [22, 556, 78]; + return verifyOperation(ReadOnlyIntArray, numbers, schema); +} + +@test:Config { + groups: ["array", "string"] +} +public isolated function testStringArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "stringArray", + "namespace": "data", + "items": "string" + }`; + + string[] colors = ["red", "green", "blue"]; + return verifyOperation(StringArray, colors, schema); +} + +@test:Config { + groups: ["array", "string"] +} +public isolated function testReadOnlyStringArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "stringArray", + "namespace": "data", + "items": "string" + }`; + + ReadOnlyStringArray colors = ["red", "green", "blue"]; + return verifyOperation(ReadOnlyStringArray, colors, schema); +} + +@test:Config { + groups: ["array", "string"] +} +public isolated function testArrayOfStringArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "stringArray", + "namespace": "data", + "items": { + "type": "array", + "name" : "strings", + "namespace": "data", + "items": "string" + } + }`; + + string[][] colors = [["red", "green", "blue"]]; + return verifyOperation(String2DArray, colors, schema); +} + +@test:Config { + groups: ["array", "string", "mn"] +} +public isolated function testReadOnlyArrayOfStringArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "stringArray", + "namespace": "data", + "items": { + "type": "array", + "name" : "strings", + "namespace": "data", + "items": "string" + } + }`; + + string[][] & readonly colors = [["red", "green", "blue"]]; + return verifyOperation(ReadOnlyString2DArray, colors, schema); +} + +@test:Config { + groups: ["array", "string", "enn"] +} +public isolated function testEnumArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "enums", + "namespace": "data", + "items": { + "type": "enum", + "name": "Numbers", + "symbols": [ "ONE", "TWO", "THREE", "FOUR" ] + } + }`; + + Numbers[] colors = ["ONE", "TWO", "THREE"]; + return verifyOperation(EnumArray, colors, schema); +} + +@test:Config { + groups: ["array", "string"] +} +public isolated function testArrayOfEnumArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "enums", + "namespace": "data", + "items": { + "type": "array", + "name" : "enumsValues", + "namespace": "data", + "items": { + "type": "enum", + "name": "Numbers", + "symbols": [ "ONE", "TWO", "THREE", "FOUR" ] + } + } + }`; + + Numbers[][] colors = [["ONE", "TWO", "THREE"], ["ONE", "TWO", "THREE"]]; + return verifyOperation(Enum2DArray, colors, schema); +} + +@test:Config { + groups: ["array", "float"] +} +public isolated function testFloatArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "floatArray", + "namespace": "data", + "items": "float" + }`; + + float[] numbers = [22.4, 556.84350, 78.0327]; + return verifyOperation(FloatArray, numbers, schema); +} + +@test:Config { + groups: ["array", "double"] +} +public isolated function testDoubleArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "doubleArray", + "namespace": "data", + "items": "double" + }`; + + float[] numbers = [22.439475948, 556.843549485340, 78.032985693457]; + return verifyOperation(FloatArray, numbers, schema); +} + +@test:Config { + groups: ["array", "long"] +} +public isolated function testLongArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "longArray", + "namespace": "data", + "items": "long" + }`; + + int[] numbers = [223432, 55423326, 7823423]; + return verifyOperation(IntArray, numbers, schema); +} + +@test:Config { + groups: ["array", "errors"] +} +public isolated function testInvalidDecimalArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "decimalArray", + "namespace": "data", + "items": "double" + }`; + + decimal[] numbers = [22.439475948, 556.843549485340, 78.032985693457]; + + Schema avro = check new (schema); + byte[]|Error serializedValue = avro.toAvro(numbers); + test:assertTrue(serializedValue is Error); +} + +@test:Config { + groups: ["array", "boolean"] +} +public isolated function testBooleanArrays() returns error? { + string schema = string ` + { + "type": "array", + "name" : "booleanArray", + "namespace": "data", + "items": "boolean" + }`; + + boolean[] numbers = [true, true, false]; + return verifyOperation(BooleanArray, numbers, schema); +} + +@test:Config { + groups: ["array", "error", "anydata"] +} +public isolated function testArraysWithAnydata() returns error? { + string schema = string ` + { + "type": "array", + "name" : "floatArray", + "namespace": "data", + "items": "bytes" + }`; + + anydata numbers = ["22.4".toBytes(), "556.84350", 78.0327]; + Schema avro = check new (schema); + byte[]|Error serializedValue = avro.toAvro(numbers); + test:assertTrue(serializedValue is Error); +} + +@test:Config { + groups: ["array", "byte"] +} +public isolated function testArraysWithFixed() returns error? { + string schema = string ` + { + "type": "array", + "name" : "fixedArray", + "namespace": "data", + "items": { + "type": "fixed", + "name": "FixedBytes", + "size": 2 + } + }`; + + byte[][] numbers = ["22".toBytes(), "55".toBytes(), "78".toBytes()]; + return verifyOperation(ArrayOfByteArray, numbers, schema); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testRecordsInArrays() returns error? { + string jsonFileName = string `tests/resources/schema_array_records.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + Student[] students = [{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }]; + return verifyOperation(StudentArray, students, schema); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testRecordsInReadOnlyArrays() returns error? { + string jsonFileName = string `tests/resources/schema_array_readonly_records.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + Student[] & readonly students = [{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }]; + return verifyOperation(ReadOnlyStudentArray, students, schema); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testRecordArraysInArrays() returns error? { + string jsonFileName = string `tests/resources/schema_array_record_arrays.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + Student[][] students = [[{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }], [{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }]]; + return verifyOperation(Student2DArray, students, schema); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testRecordArraysInReadOnlyArrays() returns error? { + string jsonFileName = string `tests/resources/schema_array_readonly_arrays.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + Student[][] & readonly students = [[{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }], [{ + name: "Liam", + subject: "geology" + }, { + name: "John", + subject: "math" + }]]; + + return verifyOperation(ReadOnlyStudent2DArray, students, schema); +} diff --git a/ballerina/tests/byte_tests.bal b/ballerina/tests/byte_tests.bal new file mode 100644 index 0000000..5c7ab38 --- /dev/null +++ b/ballerina/tests/byte_tests.bal @@ -0,0 +1,92 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; +import ballerina/io; + +@test:Config{ + groups: ["record", "bytes"] +} +public isolated function testRecordsWithBytes() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "favorite_color", "type": "bytes"} + ] + }`; + + Student1 student = { + name: "Liam", + favorite_color: "yellow".toBytes() + }; + return verifyOperation(Student1, student, schema); +} + +@test:Config { + groups: ["array", "byte"] +} +public isolated function testArraysWithBytes() returns error? { + string schema = string ` + { + "type": "array", + "name" : "floatArray", + "namespace": "data", + "items": "bytes" + }`; + + byte[][] numbers = ["22.4".toBytes(), "556.84350".toBytes(), "78.0327".toBytes()]; + return verifyOperation(ArrayOfByteArray, numbers, schema); +} + +@test:Config { + groups: ["primitive", "bytes"] +} +public isolated function testBytes() returns error? { + string schema = string ` + { + "type": "bytes", + "name" : "byteValue", + "namespace": "data" + }`; + + byte[] value = "5".toBytes(); + return verifyOperation(ByteArray, value, schema); +} + +@test:Config { + groups: ["record", "map", "bytes"] +} +public isolated function testNestedRecordsWithBytes() returns error? { + string jsonFileName = string `tests/resources/schema_bytes.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + Lecturer4 lecturer4 = { + name: { + "John": 1, + "Sam": 2, + "Liam": 3 + }, + byteData: "s".toBytes().cloneReadOnly(), + instructor: { + byteData: "ddd".toBytes().cloneReadOnly() + } + }; + return verifyOperation(Lecturer4, lecturer4, schema); +} diff --git a/ballerina/tests/map_tests.bal b/ballerina/tests/map_tests.bal new file mode 100644 index 0000000..5a9d553 --- /dev/null +++ b/ballerina/tests/map_tests.bal @@ -0,0 +1,772 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; +import ballerina/io; + +@test:Config { + groups: ["map", "bytes"] +} +public isolated function testMapsWithBytes() returns error? { + string schema = string ` + { + "type": "map", + "values" : "bytes", + "default": {} + }`; + + map colors = {"red": "0".toBytes(), "green": "1".toBytes(), "blue": "2".toBytes()}; + return verifyOperation(ByteArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "fixed"] +} +public isolated function testMapsWithFixed() returns error? { + string schema = string ` + { + "type": "map", + "values" : { + "type": "fixed", + "name": "name", + "size": 1 + }, + "default": {} + }`; + + map colors = {"red": "0".toBytes(), "green": "1".toBytes(), "blue": "2".toBytes()}; + return verifyOperation(ByteArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "fixed"] +} +public isolated function testMapsOfFixedMaps() returns error? { + string schema = string ` + { + "type": "map", + "values" : { + "type": "map", + "values" : { + "type": "fixed", + "name": "name", + "size": 1 + }, + "default": {} + }, + "default": {} + }`; + + map> colors = { + "red": {"r": "0".toBytes(), "g": "1".toBytes(), "b": "2".toBytes()}, + "green": {"r": "0".toBytes(), "g": "1".toBytes(), "b": "2".toBytes()}, + "blue": {"r": "0".toBytes(), "g": "1".toBytes(), "b": "2".toBytes()} + }; + return verifyOperation(MapOfByteArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "record", "kkk"] +} +public isolated function testReadOnlyMapsWithReadOnlyRecords() returns error? { + string jsonFileName = string `tests/resources/schema_map_readonly.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + map & readonly instructors = { + "john": {name: "John", student: {name: "Alice", subject: "Math"}}, + "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, + "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} + }; + + return verifyOperation(ReadOnlyMapOfReadOnlyRecord, instructors, schema); +} + +@test:Config { + groups: ["map", "record", "kkk"] +} +public isolated function testMapsWithRecordsWithReadOnly() returns error? { + string jsonFileName = string `tests/resources/schema_map_records_readonly.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + map & readonly instructors = { + "john": {name: "John", student: {name: "Alice", subject: "Math"}}, + "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, + "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} + }; + return verifyOperation(ReadOnlyMapOfRecord, instructors, schema); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithReadOnlyRecordsWithReadOnly() returns error? { + string jsonFileName = string `tests/resources/schema_map_readonly_records_readonly.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + map & readonly instructors = { + "john": {name: "John", student: {name: "Alice", subject: "Math"}}, + "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, + "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} + }; + return verifyOperation(ReadOnlyMapOfReadOnlyRecord, instructors, schema); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithReadOnlyRecords() returns error? { + string jsonFileName = string `tests/resources/schema_map_readonly_records.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + map instructors = { + "john": {name: "John", student: {name: "Alice", subject: "Math"}}, + "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, + "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} + }; + return verifyOperation(ReadOnlyMapOfRecord, instructors, schema); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithRecords() returns error? { + string jsonFileName = string `tests/resources/schema_map_records.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + map instructors = { + "john": {name: "John", student: {name: "Alice", subject: "Math"}}, + "doe": {name: "Doe", student: {name: "Bob", subject: "Science"}}, + "jane": {name: "Jane", student: {name: "Charlie", subject: "English"}} + }; + return verifyOperation(RecordMap, instructors, schema); +} + +@test:Config { + groups: ["map"] +} +public isolated function testMapsWithInt() returns error? { + string schema = string ` + { + "type": "map", + "values" : "int", + "default": {} + }`; + + map colors = {"red": 0, "green": 1, "blue": 2}; + return verifyOperation(IntMap, colors, schema); +} + +@test:Config { + groups: ["map", "enum"] +} +public isolated function testMapsWithEnum() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO", "THREE"] + }, + "default": {} + }`; + + map colors = {"red": "ONE", "green": "TWO", "blue": "THREE"}; + return verifyOperation(EnumMap, colors, schema); +} + +@test:Config { + groups: ["map", "enum", "array"] +} +public isolated function testMapsWithEnumArrays() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "name" : "enums", + "namespace": "data", + "items": { + "type": "enum", + "name": "Numbers", + "symbols": [ "ONE", "TWO", "THREE", "FOUR" ] + } + }, + "default": {} + }`; + + map colors = {"red": ["ONE", "TWO", "THREE"], "green": ["ONE", "TWO", "THREE"], "blue": ["ONE", "TWO", "THREE"]}; + return verifyOperation(EnumArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "float"] +} +public isolated function testMapsWithFloat() returns error? { + string schema = string ` + { + "type": "map", + "values": "float", + "default": {} + }`; + + map colors = {"red": 2.3453, "green": 435.563, "blue": 20347.23}; + return verifyOperation(FloatMap, colors, schema); +} + +@test:Config { + groups: ["map", "double"] +} +public isolated function testMapsWithDouble() returns error? { + string schema = string ` + { + "type": "map", + "values": "double", + "default": {} + }`; + + map colors = {"red": 2.3453, "green": 435.563, "blue": 20347.23}; + return verifyOperation(FloatMap, colors, schema); +} + +@test:Config { + groups: ["map", "double", "array"] +} +public isolated function testMapsWithDoubleArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "name" : "enums", + "namespace": "data", + "items": "double" + }, + "default": {} + }`; + + map colors = {"red": [2.3434253, 435.56433, 20347.22343], "green": [2.3452343, 435.56343, 20347.2423], "blue": [2.3453243, 435.56243, 20347.22343]}; + return verifyOperation(FloatArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "long"] +} +public isolated function testMapsWithLong() returns error? { + string schema = string ` + { + "type": "map", + "values": "long", + "default": {} + }`; + + map colors = {"red": 2, "green": 435, "blue": 2034723}; + return verifyOperation(IntMap, colors, schema); +} + +@test:Config { + groups: ["map", "string"] +} +public isolated function testMapsWithStrings() returns error? { + string schema = string ` + { + "type": "map", + "values": "string", + "default": {} + }`; + + map colors = {"red": "2", "green": "435", "blue": "2034723"}; + return verifyOperation(StringMap, colors, schema); +} + +@test:Config { + groups: ["map", "errors"] +} +public isolated function testMapsWithUnionTypes() returns error? { + string schema = string ` + { + "type": "map", + "values": ["string", "int"], + "default": {} + }`; + + map colors = {"red": "2", "green": "435", "blue": "2034723"}; + + Schema avro = check new (schema); + byte[]|Error serializedValue = avro.toAvro(colors); + test:assertTrue(serializedValue is Error); +} + +@test:Config { + groups: ["map", "boolean"] +} +public isolated function testMapsWithBoolean() returns error? { + string schema = string ` + { + "type": "map", + "values": "boolean", + "default": {} + }`; + + map colors = {"red": true, "green": false, "blue": false}; + return verifyOperation(BooleanMap, colors, schema); +} + +@test:Config { + groups: ["map", "boolean", "ssq"] +} +public isolated function testMapsWithBooleanWithReadOnlyValues() returns error? { + string schema = string ` + { + "type": "map", + "values": "boolean", + "default": {} + }`; + + map & readonly colors = {"red": true, "green": false, "blue": false}; + return verifyOperation(ReadOnlyBooleanMap, colors, schema); +} + +@test:Config { + groups: ["map"] +} +public isolated function testMapsWithMaps() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "map", + "values": "long" + }, + "default": {} + }`; + + map> colors = { + "red": {"r": 2, "g": 3, "b": 4}, + "green": {"r": 5, "g": 6, "b": 7}, + "blue": {"r": 8, "g": 9, "b": 10} + }; + return verifyOperation(MapOfIntMap, colors, schema); +} + +@test:Config { + groups: ["map", "k"] +} +public isolated function testMapsWithNestedMapsWithReadOnlyValues() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "map", + "values": { + "type": "map", + "values": "int" + } + }, + "default": {} + }`; + + map>> & readonly colors = { + "red": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}}, + "green": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}}, + "blue": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}} + }; + return verifyOperation(ReadOnlyMapOfReadOnlyMap, colors, schema); +} + +@test:Config { + groups: ["map", "k"] +} +public isolated function testMapsWithNestedMaps() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "map", + "values": { + "type": "map", + "values": "int" + } + }, + "default": {} + }`; + + map>> colors = { + "red": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}}, + "green": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}}, + "blue": {"r": {"r": 2, "g": 3, "b": 4}, "g": {"r": 5, "g": 6, "b": 7}, "b": {"r": 8, "g": 9, "b": 10}} + }; + return verifyOperation(MapOfMap, colors, schema); +} + +@test:Config { + groups: ["map", "long", "qwe"] +} +public isolated function testMapsWithLongArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "long" + }, + "default": {} + }`; + + map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; + return verifyOperation(IntArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "int", "az"] +} +public isolated function testMapsWithIntArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "int" + }, + "default": {} + }`; + + map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; + return verifyOperation(IntArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "int"] +} +public isolated function testMapsWithIntArrayWithReadOnlyValues() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "int" + }, + "default": {} + }`; + + map & readonly colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; + return verifyOperation(ReadOnlyIntArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "float"] +} +public isolated function testMapsWithFloatArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "float" + }, + "default": {} + }`; + + map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; + return verifyOperation(FloatArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "string"] +} +public isolated function testMapsWithStringArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "string" + }, + "default": {} + }`; + + map colors = {"red": ["252", "122", "41"], "green": ["235", "163", "23"], "blue": ["207", "123"]}; + return verifyOperation(StringArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "string", "null", "union", "sbs"] +} +public isolated function testMapsWithUnionArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": ["string", "null"] + }, + "default": {} + }`; + + map colors = {"red": ["252", "122", "41"], "green": ["235", "163", "23"], "blue": ["207", "123"]}; + return verifyOperation(StringArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "int", "null", "union"] +} +public isolated function testMapsWithUnionIntArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": ["int", "null"] + }, + "default": {} + }`; + + map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; + return verifyOperation(IntArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "long", "null", "union", "sxs"] +} +public isolated function testMapsWithUnionLongArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": ["long", "null"] + }, + "default": {} + }`; + + map colors = {"red": [252, 122, 41], "green": [235, 163, 23], "blue": [207, 123]}; + return verifyOperation(IntArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "float", "null", "union", "sss"] +} +public isolated function testMapsWithUnionFloatArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": ["float", "null"] + }, + "default": {} + }`; + + map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; + return verifyOperation(FloatArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "float", "null", "union"] +} +public isolated function testMapsWithUnionDoubleArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": ["double", "null"] + }, + "default": {} + }`; + + map colors = {"red": [252.32, 122.45, 41.342], "green": [235.321, 163.3, 23.324], "blue": [207.23434, 123.23]}; + return verifyOperation(FloatArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "bytes"] +} +public isolated function testMapsWithBytesArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "bytes" + }, + "default": {} + }`; + + map colors = {"red": ["252".toBytes(), "122".toBytes(), "41".toBytes()], "green": ["235".toBytes(), "163".toBytes(), "23".toBytes()], "blue": ["207".toBytes(), "123".toBytes()]}; + return verifyOperation(ByteArrayArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "bytes"] +} +public isolated function testMapsWithFixedArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": { + "type": "fixed", + "name": "name", + "size": 3 + } + }, + "default": {} + }`; + + map colors = {"red": ["252".toBytes(), "122".toBytes(), "411".toBytes()], "green": ["235".toBytes(), "163".toBytes(), "213".toBytes()], "blue": ["207".toBytes(), "123".toBytes()]}; + return verifyOperation(ByteArrayArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "boolean"] +} +public isolated function testMapsWithBooleanArray() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "items": "boolean" + }, + "default": {} + }`; + + map colors = {"red": [true, false, true], "green": [false, true, false], "blue": [true, false]}; + return verifyOperation(BooleanArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithArrayOfRecordArray() returns error? { + string jsonFileName = string `tests/resources/schema_map_array_record_arrays.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + map instructors = { + "john": [[{name: "John", student: {name: "Alice", subject: "Math"}}, {name: "John", student: {name: "Alice", subject: "Math"}}]], + "doe": [[{name: "Doe", student: {name: "Bob", subject: "Science"}}, {name: "Doe", student: {name: "Bob", subject: "Science"}}]], + "jane": [[{name: "Jane", student: {name: "Charlie", subject: "English"}}, {name: "Jane", student: {name: "Charlie", subject: "English"}}]] + }; + return verifyOperation(MapOfArrayOfRecordArray, instructors, schema); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testArrayOfStringArrayMaps() returns error? { + string schema = string ` + { + "type": "map", + "values": { + "type": "array", + "name" : "stringArray", + "namespace": "data", + "items": { + "type": "array", + "name" : "strings", + "namespace": "data", + "items": "string" + } + } + }`; + + map colors = { + "red": [["red", "green", "blue"], ["red", "green", "blue"], ["red", "green", "blue"]], + "green": [["red", "green", "blue"], ["red", "green", "blue"], ["red", "green", "blue"]], + "blue": [["red", "green", "blue"], ["red", "green", "blue"], ["red", "green", "blue"]] + }; + return verifyOperation(StringArrayArrayMap, colors, schema); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithNestedRecordMaps() returns error? { + string jsonFileName = string `tests/resources/schema_map_record_maps.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + Lecturer lec = { + name: "John", + instructor: { + name: "Jane", + student: { + name: "Charlie", + subject: "English" + } + } + }; + + map> lecturers = { + "john": {"john": lec, "doe": lec}, + "doe": {"john": lec, "doe": lec}, + "jane": {"john": lec, "doe": lec} + }; + return verifyOperation(MapOfRecordMap, lecturers, schema); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithNestedRecordArrayMaps() returns error? { + string jsonFileName = string `tests/resources/schema_map_record_array.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + Lecturer[] lecs = [{name: "John", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}, + {name: "Doe", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}]; + + map> lecturers = { + "john": {"r": lecs, "g": lecs, "b": lecs}, + "doe": {"r": lecs, "g": lecs, "b": lecs}, + "jane": {"r": lecs, "g": lecs, "b": lecs} + }; + return verifyOperation(MapOfRecordArrayMap, lecturers, schema); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithRecordArray() returns error? { + string jsonFileName = string `tests/resources/schema_map_array_record.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + map instructors = { + "john": [{name: "John", student: {name: "Alice", subject: "Math"}}, {name: "John", student: {name: "Alice", subject: "Math"}}], + "doe": [{name: "Doe", student: {name: "Bob", subject: "Science"}}, {name: "Doe", student: {name: "Bob", subject: "Science"}}], + "jane": [{name: "Jane", student: {name: "Charlie", subject: "English"}}, {name: "Jane", student: {name: "Charlie", subject: "English"}}] + }; + return verifyOperation(MapOfRecordArray, instructors, schema); +} + +@test:Config { + groups: ["map", "record"] +} +public isolated function testMapsWithNestedRecordArrayReadOnlyMaps() returns error? { + string jsonFileName = string `tests/resources/schema_map_record_array_readonly.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + Lecturer[] lecs = [{name: "John", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}, + {name: "Doe", instructor: {name: "Jane", student: {name: "Charlie", subject: "English"}}}]; + + map mapValue = {"r": lecs, "g": lecs, "b": lecs}; + map> & readonly lecturers = { + "john": mapValue.cloneReadOnly(), + "doe": mapValue.cloneReadOnly(), + "jane": mapValue.cloneReadOnly() + }; + return verifyOperation(ReadOnlyMapOfRecordArray, lecturers, schema); +} diff --git a/ballerina/tests/primitive_tests.bal b/ballerina/tests/primitive_tests.bal new file mode 100644 index 0000000..0b7add8 --- /dev/null +++ b/ballerina/tests/primitive_tests.bal @@ -0,0 +1,136 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; + +@test:Config { + groups: ["primitive", "int"] +} +public isolated function testIntValue() returns error? { + string schema = string ` + { + "type": "int", + "name" : "intValue", + "namespace": "data" + }`; + + int value = 5; + return verifyOperation(int, value, schema); +} + +@test:Config { + groups: ["primitive", "float"] +} +public isolated function testFloatValue() returns error? { + string schema = string ` + { + "type": "float", + "name" : "floatValue", + "namespace": "data" + }`; + + float value = 5.5; + return verifyOperation(float, value, schema); +} + +@test:Config { + groups: ["primitive", "double"] +} +public isolated function testDoubleValue() returns error? { + string schema = string ` + { + "type": "double", + "name" : "doubleValue", + "namespace": "data" + }`; + + float value = 5.5595; + return verifyOperation(float, value, schema); +} + +@test:Config { + groups: ["primitive", "check", "l"] +} +public isolated function testLongValue() returns error? { + string schema = string ` + { + "type": "long", + "name" : "longValue", + "namespace": "data" + }`; + + int value = 555950000000000000; + return verifyOperation(int, value, schema); +} + +@test:Config { + groups: ["primitive", "check"] +} +public isolated function testStringValue() returns error? { + string schema = string ` + { + "type": "string", + "name" : "stringValue", + "namespace": "data" + }`; + + string value = "test"; + return verifyOperation(string, value, schema); +} + +@test:Config { + groups: ["primitive", "check"] +} +public isolated function testBoolean() returns error? { + string schema = string ` + { + "type": "boolean", + "name" : "booleanValue", + "namespace": "data" + }`; + + boolean value = true; + return verifyOperation(boolean, value, schema); +} + +@test:Config { + groups: ["primitive", "null"] +} +public isolated function testNullValues() returns error? { + string schema = string ` + { + "type": "null", + "name" : "nullValue", + "namespace": "data" + }`; + return verifyOperation(NullType, (), schema); +} + +@test:Config { + groups: ["primitive", "null"] +} +public isolated function testNullValuesWithNonNullData() returns error? { + string schema = string ` + { + "type": "null", + "name" : "nullValue", + "namespace": "data" + }`; + + Schema avro = check new (schema); + byte[]|error serializedValue = avro.toAvro("string"); + test:assertTrue(serializedValue is error); +} diff --git a/ballerina/tests/record_tests.bal b/ballerina/tests/record_tests.bal new file mode 100644 index 0000000..bc26942 --- /dev/null +++ b/ballerina/tests/record_tests.bal @@ -0,0 +1,372 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; +import ballerina/io; + +@test:Config { + groups: ["record"] +} +public isolated function testRecords() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "subject", "type": "string"} + ] + }`; + + Student student = { + name: "Liam", + subject: "geology" + }; + return verifyOperation(Student, student, schema); +} + +@test:Config { + groups: ["record"] +} +public isolated function testRecordsWithDifferentTypeOfFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "int"} + ] + }`; + + Person student = { + name: "Liam", + age: 52 + }; + return verifyOperation(Person, student, schema); +} + +@test:Config { + groups: ["record"] +} +public isolated function testNestedRecords() returns error? { + string jsonFileName = string `tests/resources/schema_record_nested.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + Lecturer3 lecturer = { + name: {"John": 1, "Sam": 2, "Liam": 3}, + age: 11, + instructor: { + name: "Liam", + student: { + name: "Sam", + subject: "geology" + } + } + }; + + return verifyOperation(Lecturer3, lecturer, schema); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testArraysInRecords() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "colors", "type": {"type": "array", "items": "string"}} + ] + }`; + + Color colors = { + name: "Red", + colors: ["maroon", "dark red", "light red"] + }; + + return verifyOperation(Color, colors, schema); +} + +@test:Config { + groups: ["record", "array"] +} +public isolated function testArraysInReadOnlyRecords() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "colors", "type": {"type": "array", "items": "string"}} + ] + }`; + + ReadOnlyColors colors = { + name: "Red", + colors: ["maroon", "dark red", "light red"] + }; + + return verifyOperation(ReadOnlyColors, colors, schema); +} + +@test:Config { + groups: ["record", "errors"] +} +public isolated function testArraysInRecordsWithInvalidSchema() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "colors", "type": "bytes"} + ] + }`; + + Color1 colors = { + name: "Red", + colors: "ss".toBytes() + }; + + Schema avroProducer = check new (schema); + byte[] serialize = check avroProducer.toAvro(colors); + string schema2 = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "colors", "type": {"type": "array", "items": "int"}} + ] + }`; + Schema avroConsumer = check new (schema2); + Color1|Error deserializedValue = avroConsumer.fromAvro(serialize); + test:assertTrue(deserializedValue is Error); +} + +@test:Config { + groups: ["record", "union"] +} +public isolated function testRecordsWithStringRecordUnionType() returns error? { + string jsonFileName = string `tests/resources/schema_record_string_record_union.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + MultipleUnionRecord course = { + name: "data", + value: 0.0, + credits: 5, + student: string `{name: "Jon", subject: "geo"}adsk` + }; + + return verifyOperation(MultipleUnionRecord, course, schema); +} + +@test:Config { + groups: ["record", "union"] +} +public isolated function testRecordsWithUnionTypes() returns error? { + string jsonFileName = string `tests/resources/schema_record_union.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + UnionRecord course = { + name: "data", + value: 0.0, + credits: 5, + student: {name: "Jon", subject: "geo"} + }; + return verifyOperation(UnionRecord, course, schema); +} + +@test:Config { + groups: ["record", "primitive", "int"] +} +public isolated function testRecordsWithIntFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "int"} + ] + }`; + + Person student = { + name: "Liam", + age: 52 + }; + return verifyOperation(Person, student, schema); +} + +@test:Config { + groups: ["record", "primitive", "long"] +} +public isolated function testRecordsWithLongFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "long"} + ] + }`; + + Person student = { + name: "Liam", + age: 52 + }; + return verifyOperation(Person, student, schema); +} + +@test:Config { + groups: ["record", "primitive", "float"] +} +public isolated function testRecordsWithFloatFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "float"} + ] + }`; + + Students student = { + name: "Liam", + age: 52.656 + }; + return verifyOperation(Students, student, schema); +} + +@test:Config { + groups: ["record", "primitive", "double"] +} +public isolated function testRecordsWithDoubleFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "age", "type": "double"} + ] + }`; + + Students student = { + name: "Liam", + age: 52.656 + }; + return verifyOperation(Students, student, schema); +} + +@test:Config { + groups: ["record", "primitive", "boolean"] +} +public isolated function testRecordsWithBooleanFields() returns error? { + string schema = string ` + { + "namespace": "example.avro", + "type": "record", + "name": "Student", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "under19", "type": "boolean"} + ] + }`; + + StudentRec student = { + name: "Liam", + under19: false + }; + return verifyOperation(StudentRec, student, schema); +} + +@test:Config { + groups: ["record", "union"] +} +public isolated function testOptionalValuesInRecords() returns error? { + string jsonFileName = string `tests/resources/schema_record_optional_fields.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + + Instructor instructor = { + name: "John", + student: { + name: "Alice", + subject: "Math" + } + }; + + Lecturer5 lecturer5 = { + name: { + "John": 1, + "Sam": 2, + "Liam": 3 + }, + bytes: "ss".toBytes(), + instructorClone: instructor.cloneReadOnly(), + instructors: instructor + }; + return verifyOperation(Lecturer5, lecturer5, schema); +} + +@test:Config { + groups: ["record", "union"] +} +public isolated function testOptionalMultipleFieldsInRecords() returns error? { + string jsonFileName = string `tests/resources/schema_record_multiple_optional_fields.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); + Instructor instructor = { + name: "John", + student: { + name: "Alice", + subject: "Math" + } + }; + + Numbers number = ONE; + + Lecturer6 lecturer6 = { + temporary: false, + maps: { + "1": 100, + "2": 200 + }, + bytes: "ss".toBytes(), + age: 30, + number: number, + name: "Lecturer Name", + floatNumber: 123.45, + colors: [number, number, number], + instructorClone: instructor.cloneReadOnly(), + instructors: instructor + }; + return verifyOperation(Lecturer6, lecturer6, schema); +} diff --git a/ballerina/tests/resources/schema_array_readonly_arrays.json b/ballerina/tests/resources/schema_array_readonly_arrays.json new file mode 100644 index 0000000..dfb920b --- /dev/null +++ b/ballerina/tests/resources/schema_array_readonly_arrays.json @@ -0,0 +1,22 @@ +{ + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "array", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + } +} diff --git a/ballerina/tests/resources/schema_array_readonly_records.json b/ballerina/tests/resources/schema_array_readonly_records.json new file mode 100644 index 0000000..6412977 --- /dev/null +++ b/ballerina/tests/resources/schema_array_readonly_records.json @@ -0,0 +1,19 @@ +{ + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } +} diff --git a/ballerina/tests/resources/schema_array_record_arrays.json b/ballerina/tests/resources/schema_array_record_arrays.json new file mode 100644 index 0000000..dfb920b --- /dev/null +++ b/ballerina/tests/resources/schema_array_record_arrays.json @@ -0,0 +1,22 @@ +{ + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "array", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + } +} diff --git a/ballerina/tests/resources/schema_array_records.json b/ballerina/tests/resources/schema_array_records.json new file mode 100644 index 0000000..6412977 --- /dev/null +++ b/ballerina/tests/resources/schema_array_records.json @@ -0,0 +1,19 @@ +{ + "type": "array", + "name" : "recordArray", + "namespace": "data", + "items": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } +} diff --git a/ballerina/tests/resources/schema_bytes.json b/ballerina/tests/resources/schema_bytes.json new file mode 100644 index 0000000..8daf0fa --- /dev/null +++ b/ballerina/tests/resources/schema_bytes.json @@ -0,0 +1,30 @@ +{ + "type": "record", + "name": "Lecturer4", + "fields": [ + { + "name": "name", + "type": { + "type": "map", + "values": "int" + } + }, + { + "name": "byteData", + "type": "bytes" + }, + { + "name": "instructor", + "type": { + "type": "record", + "name": "ByteRecord", + "fields": [ + { + "name": "byteData", + "type": "bytes" + } + ] + } + } + ] +} diff --git a/ballerina/tests/resources/schema_map_array_record.json b/ballerina/tests/resources/schema_map_array_record.json new file mode 100644 index 0000000..8ecb83c --- /dev/null +++ b/ballerina/tests/resources/schema_map_array_record.json @@ -0,0 +1,33 @@ +{ + "type": "map", + "values": { + "type": "array", + "items": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + } + } +} diff --git a/ballerina/tests/resources/schema_map_array_record_arrays.json b/ballerina/tests/resources/schema_map_array_record_arrays.json new file mode 100644 index 0000000..313cee1 --- /dev/null +++ b/ballerina/tests/resources/schema_map_array_record_arrays.json @@ -0,0 +1,37 @@ +{ + "type": "map", + "values": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + } + } + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_map_readonly.json b/ballerina/tests/resources/schema_map_readonly.json new file mode 100644 index 0000000..d85ea0b --- /dev/null +++ b/ballerina/tests/resources/schema_map_readonly.json @@ -0,0 +1,31 @@ +{ + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_map_readonly_records.json b/ballerina/tests/resources/schema_map_readonly_records.json new file mode 100644 index 0000000..d85ea0b --- /dev/null +++ b/ballerina/tests/resources/schema_map_readonly_records.json @@ -0,0 +1,31 @@ +{ + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_map_readonly_records_readonly.json b/ballerina/tests/resources/schema_map_readonly_records_readonly.json new file mode 100644 index 0000000..d85ea0b --- /dev/null +++ b/ballerina/tests/resources/schema_map_readonly_records_readonly.json @@ -0,0 +1,31 @@ +{ + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_map_record_array.json b/ballerina/tests/resources/schema_map_record_array.json new file mode 100644 index 0000000..70de00a --- /dev/null +++ b/ballerina/tests/resources/schema_map_record_array.json @@ -0,0 +1,49 @@ +{ + "type": "map", + "values": { + "type": "map", + "values": { + "type": "array", + "items": { + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "instructor", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "subject", + "type": ["null", "string"] + } + ] + }] + } + ] + }] + } + ] + } + } + } +} diff --git a/ballerina/tests/resources/schema_map_record_array_readonly.json b/ballerina/tests/resources/schema_map_record_array_readonly.json new file mode 100644 index 0000000..70de00a --- /dev/null +++ b/ballerina/tests/resources/schema_map_record_array_readonly.json @@ -0,0 +1,49 @@ +{ + "type": "map", + "values": { + "type": "map", + "values": { + "type": "array", + "items": { + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "instructor", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "subject", + "type": ["null", "string"] + } + ] + }] + } + ] + }] + } + ] + } + } + } +} diff --git a/ballerina/tests/resources/schema_map_record_maps.json b/ballerina/tests/resources/schema_map_record_maps.json new file mode 100644 index 0000000..8d572f2 --- /dev/null +++ b/ballerina/tests/resources/schema_map_record_maps.json @@ -0,0 +1,46 @@ +{ + "type": "map", + "values": { + "type": "map", + "values": { + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "instructor", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "subject", + "type": ["null", "string"] + } + ] + }] + } + ] + }] + } + ] + } + } +} diff --git a/ballerina/tests/resources/schema_map_records.json b/ballerina/tests/resources/schema_map_records.json new file mode 100644 index 0000000..d85ea0b --- /dev/null +++ b/ballerina/tests/resources/schema_map_records.json @@ -0,0 +1,31 @@ +{ + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_map_records_readonly.json b/ballerina/tests/resources/schema_map_records_readonly.json new file mode 100644 index 0000000..d85ea0b --- /dev/null +++ b/ballerina/tests/resources/schema_map_records_readonly.json @@ -0,0 +1,31 @@ +{ + "type": "map", + "values": { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }, + "default": {} +} diff --git a/ballerina/tests/resources/schema_2.json b/ballerina/tests/resources/schema_nested_records.json similarity index 100% rename from ballerina/tests/resources/schema_2.json rename to ballerina/tests/resources/schema_nested_records.json diff --git a/ballerina/tests/resources/schema_record_multiple_optional_fields.json b/ballerina/tests/resources/schema_record_multiple_optional_fields.json new file mode 100644 index 0000000..afc77be --- /dev/null +++ b/ballerina/tests/resources/schema_record_multiple_optional_fields.json @@ -0,0 +1,95 @@ +{ + "type": "record", + "name": "Lecturer6", + "fields": [ + { + "name": "temporary", + "type": ["null", "boolean"] + }, + { + "name": "map", + "type": [ + "null", + { + "type": "map", + "values": "int" + } + ] + }, + { + "name": "number", + "type": [ + "null", + { + "type": "enum", + "name": "Numbers", + "symbols": [ "ONE", "TWO", "THREE", "FOUR" ] + } + ] + }, + { + "name": "bytes", + "type": ["null", { + "type": "fixed", + "name": "FixedBytes", + "size": 2 + }] + }, + { + "name": "age", + "type": ["null", "long"] + }, + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "floatNumber", + "type": ["null", "float"] + }, + { + "name": "colors", + "type": ["null", { + "type": "array", + "items": { + "type": "enum", + "name": "ColorEnum", + "symbols": ["ONE", "TWO", "THREE"] + } + }] + }, + { + "name": "instructorClone", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + }] + }] + }, + { + "name": "instructors", + "type": ["null", "Instructor"] + } + ] +} diff --git a/ballerina/tests/resources/schema_record_nested.json b/ballerina/tests/resources/schema_record_nested.json new file mode 100644 index 0000000..3387664 --- /dev/null +++ b/ballerina/tests/resources/schema_record_nested.json @@ -0,0 +1,49 @@ +{ + "namespace": "example.avro", + "type": "record", + "name": "Lecturer", + "fields": [ + { + "name": "name", + "type": { + "type": "map", + "values" : "int", + "default": {} + } + }, + { + "name": "age", + "type": "long" + }, + { + "name": "instructor", + "type": { + "name": "Instructor", + "type": "record", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "student", + "type": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + } + } + ] + } + } + ] +} diff --git a/ballerina/tests/resources/schema_record_optional_fields.json b/ballerina/tests/resources/schema_record_optional_fields.json new file mode 100644 index 0000000..847fbb5 --- /dev/null +++ b/ballerina/tests/resources/schema_record_optional_fields.json @@ -0,0 +1,54 @@ +{ + "type": "record", + "name": "Lecturer5", + "fields": [ + { + "name": "name", + "type": [ + "null", + { + "type": "map", + "values": "int" + } + ] + }, + { + "name": "bytes", + "type": ["null", "bytes"] + }, + { + "name": "instructorClone", + "type": ["null", { + "type": "record", + "name": "Instructor", + "fields": [ + { + "name": "name", + "type": ["null", "string"] + }, + { + "name": "student", + "type": ["null", { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "subject", + "type": "string" + } + ] + }] + } + ] + }] + }, + { + "name": "instructors", + "type": ["null", "Instructor"] + } + ] +} diff --git a/ballerina/tests/resources/schema_record_string_record_union.json b/ballerina/tests/resources/schema_record_string_record_union.json new file mode 100644 index 0000000..5c3a3d6 --- /dev/null +++ b/ballerina/tests/resources/schema_record_string_record_union.json @@ -0,0 +1,40 @@ +{ + "type": "record", + "name": "Course", + "namespace": "example.avro", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "value", + "type": "float" + }, + { + "name": "credits", + "type": ["null", "int"] + }, + { + "name": "student", + "type": [ + "null", + { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + }, + "string" + ] + } + ] +} diff --git a/ballerina/tests/resources/schema_record_union.json b/ballerina/tests/resources/schema_record_union.json new file mode 100644 index 0000000..2fa5b0c --- /dev/null +++ b/ballerina/tests/resources/schema_record_union.json @@ -0,0 +1,36 @@ +{ + "type": "record", + "name": "Course", + "namespace": "example.avro", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "value", + "type": "float" + }, + { + "name": "credits", + "type": ["null", "int"] + }, + { + "name": "student", + "type": { + "type": "record", + "name": "Student", + "fields": [ + { + "name": "name", + "type": ["string", "null"] + }, + { + "name": "subject", + "type": ["string", "null"] + } + ] + } + } + ] +} diff --git a/ballerina/tests/resources/schema_1.json b/ballerina/tests/resources/schema_records.json similarity index 100% rename from ballerina/tests/resources/schema_1.json rename to ballerina/tests/resources/schema_records.json diff --git a/ballerina/tests/resources/schema_union_enums.json b/ballerina/tests/resources/schema_union_enums.json new file mode 100644 index 0000000..583d686 --- /dev/null +++ b/ballerina/tests/resources/schema_union_enums.json @@ -0,0 +1,14 @@ +{ + "type": "record", + "name": "ExampleRecord", + "fields": [ + { + "name": "field1", + "type": ["string", { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO"] + }] + } + ] +} diff --git a/ballerina/tests/resources/schema_union_fixed.json b/ballerina/tests/resources/schema_union_fixed.json new file mode 100644 index 0000000..70c86ad --- /dev/null +++ b/ballerina/tests/resources/schema_union_fixed.json @@ -0,0 +1,17 @@ +{ + "type": "record", + "name": "ExampleRecord", + "fields": [ + { + "name": "field1", + "type": [ + "string", + { + "type": "fixed", + "name": "Fixed", + "size": 2 + } + ] + } + ] +} diff --git a/ballerina/tests/resources/schema_union_fixed_strings.json b/ballerina/tests/resources/schema_union_fixed_strings.json new file mode 100644 index 0000000..70c86ad --- /dev/null +++ b/ballerina/tests/resources/schema_union_fixed_strings.json @@ -0,0 +1,17 @@ +{ + "type": "record", + "name": "ExampleRecord", + "fields": [ + { + "name": "field1", + "type": [ + "string", + { + "type": "fixed", + "name": "Fixed", + "size": 2 + } + ] + } + ] +} diff --git a/ballerina/tests/resources/schema_union_records.json b/ballerina/tests/resources/schema_union_records.json new file mode 100644 index 0000000..e9d6f9a --- /dev/null +++ b/ballerina/tests/resources/schema_union_records.json @@ -0,0 +1,31 @@ +{ + "type": "record", + "name": "UnionRec", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "record", + "name": "UnionEnumRecord", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO"] + } + ] + } + ] + } + ] + } + ] +} diff --git a/ballerina/tests/resources/schema_union_records_strings.json b/ballerina/tests/resources/schema_union_records_strings.json new file mode 100644 index 0000000..e9d6f9a --- /dev/null +++ b/ballerina/tests/resources/schema_union_records_strings.json @@ -0,0 +1,31 @@ +{ + "type": "record", + "name": "UnionRec", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "record", + "name": "UnionEnumRecord", + "fields": [ + { + "name": "field1", + "type": [ + "null", + "string", + { + "type": "enum", + "name": "Numbers", + "symbols": ["ONE", "TWO"] + } + ] + } + ] + } + ] + } + ] +} diff --git a/ballerina/tests/test.bal b/ballerina/tests/test.bal index 3ad40ee..6309bd6 100644 --- a/ballerina/tests/test.bal +++ b/ballerina/tests/test.bal @@ -17,405 +17,155 @@ import ballerina/io; import ballerina/test; -@test:Config {} -public isolated function testEnums() returns error? { - string schema = string ` - { - "type" : "enum", - "name" : "Numbers", - "namespace": "data", - "symbols" : [ "ONE", "TWO", "THREE", "FOUR" ] - }`; - - Numbers number = "ONE"; - +public isolated function verifyOperation(typedesc providedType, + anydata value, string schema) returns error? { Schema avro = check new (schema); - byte[] encode = check avro.toAvro(number); - Numbers deserialize = check avro.fromAvro(encode); - test:assertEquals(number, deserialize); + byte[] serializedValue = check avro.toAvro(value); + var deserializedValue = check avro.fromAvro(serializedValue, providedType); + test:assertEquals(deserializedValue, value); } @test:Config { - groups: ["errors"] + groups: ["enum", "union"] } -public isolated function testEnumsWithString() returns error? { - string schema = string ` - { - "type" : "enum", - "name" : "Numbers", - "namespace": "data", - "symbols" : [ "ONE", "TWO", "THREE", "FOUR" ] - }`; +public isolated function testUnionEnums() returns error? { + string jsonFileName = string `tests/resources/schema_union_enums.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); - string number = "FIVE"; - - Schema avro = check new (schema); - byte[]|error encode = avro.toAvro(number); - test:assertTrue(encode is error); + UnionEnumRecord number = { + field1: ONE + }; + return verifyOperation(UnionEnumRecord, number, schema); } -@test:Config {} -public isolated function testMaps() returns error? { - string schema = string ` - { - "type": "map", - "values" : "int", - "default": {} - }`; - - map colors = {"red": 0, "green": 1, "blue": 2}; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - map deserialize = check avro.fromAvro(encode); - test:assertEquals(colors, deserialize); +@test:Config { + groups: ["fixed", "union"] } +public isolated function testUnionFixed() returns error? { + string jsonFileName = string `tests/resources/schema_union_fixed.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); -@test:Config {} -public isolated function testNestedRecords() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Lecturer", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "instructor", - "type": { - "name": "Instructor", - "type": "record", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "student", - "type": { - "type": "record", - "name": "Student", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "subject", - "type": "string" - } - ] - } - } - ] - } - } - ] - }`; - - Lecturer lecturer = { - name: "John", - instructor: { - name: "Liam", - student: { - name: "Sam", - subject: "geology" - } - } + UnionFixedRecord number = { + field1: "ON".toBytes() }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(lecturer); - Lecturer deserialize = check avro.fromAvro(serialize); - test:assertEquals(lecturer, deserialize); + return verifyOperation(UnionFixedRecord, number, schema); } -@test:Config {} -public isolated function testArraysInRecords() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Student", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "colors", "type": {"type": "array", "items": "string"}} - ] - }`; +@test:Config { + groups: ["fixed", "union"] +} +public isolated function testUnionFixeWithReadOnlyValues() returns error? { + string jsonFileName = string `tests/resources/schema_union_fixed_strings.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); - Color colors = { - name: "Red", - colors: ["maroon", "dark red", "light red"] + ReadOnlyUnionFixed number = { + field1: "ON".toBytes().cloneReadOnly() }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(colors); - Color deserialize = check avro.fromAvro(serialize); - test:assertEquals(colors, deserialize); + return verifyOperation(ReadOnlyUnionFixed, number, schema); } @test:Config { - groups: ["errors", "qwe"] + groups: ["fixed", "union"] } -public isolated function testArraysInRecordsWithInvalidSchema() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Student", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "colors", "type": {"type": "array", "items": "string"}} - ] - }`; +public isolated function testUnionsWithRecordsAndStrings() returns error? { + string jsonFileName = string `tests/resources/schema_union_records_strings.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); - Color colors = { - name: "Red", - colors: ["maroon", "dark red", "light red"] + UnionRec number = { + field1: { + field1: "ONE" + } }; - - Schema avroProducer = check new (schema); - byte[] serialize = check avroProducer.toAvro(colors); - - string schema2 = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Student", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "colors", "type": {"type": "array", "items": "int"}} - ] - }`; - Schema avroConsumer = check new (schema2); - Color|Error deserialize = avroConsumer.fromAvro(serialize); - test:assertTrue(deserialize is Error); + return verifyOperation(UnionRec, number, schema); } -@test:Config {} -public isolated function testRecords() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Student", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "subject", "type": "string"} - ] - }`; - - Student student = { - name: "Liam", - subject: "geology" - }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(student); - Student deserialize = check avro.fromAvro(serialize); - test:assertEquals(student, deserialize); +@test:Config { + groups: ["fixed", "union"] } +public isolated function testUnionsWithReadOnlyRecords() returns error? { + string jsonFileName = string `tests/resources/schema_union_records.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); -@test:Config {} -public isolated function testRecordsWithDifferentTypeOfFields() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Student", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "age", "type": "int"} - ] - }`; - - Person student = { - name: "Liam", - age: 52 + ReadOnlyRec number = { + field1: { + field1: "ONE".cloneReadOnly() + } }; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(student); - Person deserialize = check avro.fromAvro(encode); - test:assertEquals(student, deserialize); + return verifyOperation(ReadOnlyRec, number, schema); } -@test:Config {} -public isolated function testRecordsWithUnionTypes() returns error? { - string schema = string ` - { - "namespace": "example.avro", - "type": "record", - "name": "Course", - "fields": [ - {"name": "name", "type": ["string", "null"]}, - {"name": "credits", "type": ["int", "null"]} - ] - }`; - - Course course = { - name: (), - credits: () - }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(course); - Course deserialize = check avro.fromAvro(serialize); - test:assertEquals(course, deserialize); +@test:Config { + groups: ["enum"] } - -@test:Config {} -public isolated function testArrays() returns error? { +public isolated function testEnums() returns error? { string schema = string ` { - "type": "array", - "name" : "StringArray", + "type" : "enum", + "name" : "Numbers", "namespace": "data", - "items": "string" + "symbols" : [ "ONE", "TWO", "THREE", "FOUR" ] }`; - string[] colors = ["red", "green", "blue"]; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(colors); - string[] deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, colors); + Numbers number = "ONE"; + return verifyOperation(Numbers, number, schema); } -@test:Config {} -public isolated function testIntValue() returns error? { - string schema = string ` - { - "type": "int", - "name" : "intValue", - "namespace": "data" - }`; - - int value = 5; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - int deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); +@test:Config { + groups: ["enum"] } - -@test:Config {} -public isolated function testFloatValue() returns error? { +public isolated function testEnumsWithReadOnlyValues() returns error? { string schema = string ` { - "type": "float", - "name" : "floatValue", - "namespace": "data" + "type" : "enum", + "name" : "Numbers", + "namespace": "data", + "symbols" : [ "ONE", "TWO", "THREE", "FOUR" ] }`; - float value = 5.5; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - float deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + Numbers & readonly number = "ONE"; + return verifyOperation(Numbers, number, schema); } -@test:Config {} -public isolated function testDoubleValue() returns error? { - string schema = string ` - { - "type": "double", - "name" : "doubleValue", - "namespace": "data" - }`; - - float value = 5.5595; - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - float deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); +@test:Config { + groups: ["errors", "enum"] } - -@test:Config {} -public isolated function testLongValue() returns error? { +public isolated function testEnumsWithString() returns error? { string schema = string ` { - "type": "long", - "name" : "longValue", - "namespace": "data" + "type" : "enum", + "name" : "Numbers", + "namespace": "data", + "symbols" : [ "ONE", "TWO", "THREE", "FOUR" ] }`; - int value = 555950000000000000; + string number = "FIVE"; + Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - int deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + byte[]|Error serializedValue = avro.toAvro(number); + test:assertTrue(serializedValue is Error); } @test:Config { - groups: ["primitive"] -} -public isolated function testStringValue() returns error? { - string schema = string ` - { - "type": "string", - "name" : "stringValue", - "namespace": "data" - }`; - - string value = "test"; - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - string deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); + groups: ["fixed"] } - -@test:Config {} -public isolated function testBoolean() returns error? { +public isolated function testFixedWithInvalidSize() returns error? { string schema = string ` { - "type": "boolean", - "name" : "booleanValue", - "namespace": "data" + "type": "fixed", + "name": "name", + "size": 16 }`; - boolean value = true; - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - boolean deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, value); -} - -@test:Config {} -public isolated function testNullValues() returns error? { - string schema = string ` - { - "type": "null", - "name" : "nullValue", - "namespace": "data" - }`; + byte[] value = "u00".toBytes(); Schema avro = check new (schema); - byte[] encode = check avro.toAvro(()); - () deserializeJson = check avro.fromAvro(encode); - test:assertEquals(deserializeJson, ()); + byte[]|Error serializedValue = avro.toAvro(value); + test:assertTrue(serializedValue is Error); } -@test:Config {} -public isolated function testNullValuesWithNonNullData() returns error? { - string schema = string ` - { - "type": "null", - "name" : "nullValue", - "namespace": "data" - }`; - - Schema avro = check new (schema); - byte[]|error encode = avro.toAvro("string"); - test:assertTrue(encode is error); +@test:Config { + groups: ["fixed"] } - -@test:Config {} public isolated function testFixed() returns error? { string schema = string ` { @@ -425,14 +175,12 @@ public isolated function testFixed() returns error? { }`; byte[] value = "u00ffffffffffffx".toBytes(); - - Schema avro = check new (schema); - byte[] encode = check avro.toAvro(value); - byte[] deserialize = check avro.fromAvro(encode); - test:assertEquals(deserialize, value); + return verifyOperation(ByteArray, value, schema); } -@test:Config {} +@test:Config { + groups: ["record"] +} public function testDbSchemaWithRecords() returns error? { string schema = string ` { @@ -452,19 +200,15 @@ public function testDbSchemaWithRecords() returns error? { SchemaChangeKey changeKey = { databaseName: "my-db" }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(changeKey); - SchemaChangeKey deserialize = check avro.fromAvro(serialize); - test:assertEquals(changeKey, deserialize); - + return verifyOperation(SchemaChangeKey, changeKey, schema); } -@test:Config {} +@test:Config { + groups: ["record"] +} public function testComplexDbSchema() returns error? { - string jsonFileName = string `tests/resources/schema_1.json`; - json result = check io:fileReadJson(jsonFileName); - string schema = result.toString(); + string jsonFileName = string `tests/resources/schema_records.json`; + string schema = (check io:fileReadJson(jsonFileName)).toString(); Envelope envelope = { before: { @@ -527,17 +271,14 @@ public function testComplexDbSchema() returns error? { data_collection_order: 1 } }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(envelope); - Envelope deserialize = check avro.fromAvro(serialize); - test:assertEquals(envelope, deserialize); + return verifyOperation(Envelope, envelope, schema); } @test:Config { + groups: ["record"] } public function testComplexDbSchemaWithNestedRecords() returns error? { - string jsonFileName = string `tests/resources/schema_2.json`; + string jsonFileName = string `tests/resources/schema_nested_records.json`; json result = check io:fileReadJson(jsonFileName); string schema = result.toString(); @@ -592,7 +333,7 @@ public function testComplexDbSchemaWithNestedRecords() returns error? { FirstName: "Jane", MiddleName: "K", Gender: "F", - Language: "Spanish", + Language: (), Discreet: true, Deceased: false, IsBanned: false, @@ -651,9 +392,5 @@ public function testComplexDbSchemaWithNestedRecords() returns error? { }, MessageSource: "MessageSource" }; - - Schema avro = check new (schema); - byte[] serialize = check avro.toAvro(envelope2); - Envelope2 deserialize = check avro.fromAvro(serialize); - test:assertEquals(envelope2, deserialize); + return verifyOperation(Envelope2, envelope2, schema); } diff --git a/ballerina/tests/types.bal b/ballerina/tests/types.bal index ef73d18..0c613c9 100644 --- a/ballerina/tests/types.bal +++ b/ballerina/tests/types.bal @@ -19,6 +19,21 @@ public type Student record { string subject; }; +public type Student1 record { + string name; + byte[] favorite_color; +}; + +type Students record { + string name; + float age; +}; + +type StudentRec record { + string name; + boolean under19; +}; + public type Person record { string name; int age; @@ -31,7 +46,7 @@ public type Course record { public type Instructor record { string? name; - Student student; + Student? & readonly student; }; public type Lecturer record { @@ -39,11 +54,82 @@ public type Lecturer record { Instructor instructor; }; +public type Lecturer1 readonly & record { + string? name; + Instructor & readonly instructor; +}; + +public type Lecturer2 record { + string? name; + int age; + Instructor & readonly instructor; +}; + +public type Lecturer3 readonly & record { + map & readonly name; + int age; + Instructor & readonly instructor; +}; + +public type Lecturer4 readonly & record { + map & readonly name; + byte[] byteData; + ByteRecord? instructor; +}; + +public type Lecturer5 record { + map? & readonly name; + byte[]? bytes; + Instructor? & readonly instructorClone; + Instructor? instructors; +}; + +public type Lecturer6 record { + boolean? temporary; + map? & readonly maps; + byte[]? bytes; + int? age; + string? name; + Numbers? number; + float? floatNumber; + string[]? colors; + Instructor? & readonly instructorClone; + Instructor? instructors; +}; + +public type ByteRecord readonly & record { + byte[] byteData; +}; + +type UnionRecord record { + string? name; + int? credits; + float value; + StudentRecord? student; +}; + +type MultipleUnionRecord record { + string? name; + int? credits; + float value; + string|StudentRecord? student; +}; + +type StudentRecord record { + string? name; + string? subject; +}; + public type Color record { string? name; string[] colors; }; +type Color1 record { + string name; + byte[] colors; +}; + public type FixedRec record { byte[] fixed_field; string other_field; @@ -184,3 +270,67 @@ public type Envelope2 record { Block2? 'transaction; string? MessageSource; }; + + +public type UnionEnumRecord record { + string|Numbers? field1; +}; + +public type UnionFixedRecord record { + string|byte[]? field1; +}; + +public type UnionRec record { + string|UnionEnumRecord? field1; +}; + +public type ReadOnlyRec readonly & record { + string|UnionEnumRecord? & readonly field1; +}; + +type ReadOnlyUnionFixed UnionFixedRecord & readonly; +type ByteArray byte[]; +type ReadOnlyIntArray int[] & readonly; +type ReadOnlyStringArray string[] & readonly; +type StringArray string[]; +type String2DArray string[][]; +type ReadOnlyColors Color & readonly; +type NullType (); +type ByteArrayMap map; +type MapOfByteArrayMap map>; +type ReadOnlyMapOfReadOnlyRecord map & readonly; +type ReadOnlyMapOfRecord map & readonly; +type RecordMap map; +type IntMap map; +type EnumMap map; +type EnumArrayMap map; +type FloatMap map; +type FloatArrayMap map; +type StringMap map; +type BooleanMap map; +type ReadOnlyBooleanMap map & readonly; +type MapOfIntMap map>; +type ReadOnlyMapOfReadOnlyMap map>> & readonly; +type MapOfMap map>>; +type IntArrayMap map; +type ReadOnlyIntArrayMap map & readonly; +type StringArrayMap map; +type ByteArrayArrayMap map; +type BooleanArrayMap map; +type MapOfRecordArray map; +type MapOfArrayOfRecordArray map; +type StringArrayArrayMap map; +type MapOfRecordMap map>; +type MapOfRecordArrayMap map>; +type ReadOnlyMapOfRecordArray map> & readonly; +type ArrayOfByteArray byte[][]; +type ReadOnlyStudent2DArray Student[][] & readonly; +type Student2DArray Student[][]; +type ReadOnlyStudentArray Student[] & readonly; +type StudentArray Student[]; +type BooleanArray boolean[]; +type IntArray int[]; +type FloatArray float[]; +type EnumArray Numbers[]; +type Enum2DArray Numbers[][]; +type ReadOnlyString2DArray string[][] & readonly; diff --git a/build-config/resources/Ballerina.toml b/build-config/resources/Ballerina.toml index 4a41f89..c5e7203 100644 --- a/build-config/resources/Ballerina.toml +++ b/build-config/resources/Ballerina.toml @@ -44,9 +44,3 @@ groupId = "com.fasterxml.jackson.core" artifactId = "jackson-databind" version = "@jackson.version@" path = "./lib/jackson-databind-@jackson.version@.jar" - -[[platform.java11.dependency]] -groupId = "com.fasterxml.jackson.dataformat" -artifactId = "jackson-dataformat-avro" -version = "@jackson.version@" -path = "./lib/jackson-dataformat-avro-@jackson.version@.jar" diff --git a/build-config/spotbugs-exclude.xml b/build-config/spotbugs-exclude.xml new file mode 100644 index 0000000..68bb7b6 --- /dev/null +++ b/build-config/spotbugs-exclude.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/gradle.properties b/gradle.properties index 49e9e58..81030da 100644 --- a/gradle.properties +++ b/gradle.properties @@ -15,5 +15,4 @@ ballerinaGradlePluginVersion=2.0.1 # Dependencies stdlibIoVersion=1.6.0 avroVersion=1.11.3 -avroJacksonVersion=1.8.2 jacksonVersion=2.17.0 diff --git a/native/build.gradle b/native/build.gradle index b883954..feecbbf 100644 --- a/native/build.gradle +++ b/native/build.gradle @@ -30,8 +30,7 @@ dependencies { implementation group: 'org.ballerinalang', name: 'ballerina-lang', version: "${ballerinaLangVersion}" implementation group: 'org.ballerinalang', name: 'ballerina-runtime', version: "${ballerinaLangVersion}" - implementation group: 'org.apache.avro', name: 'avro', version: "${avroJacksonVersion}" - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-avro', version: "${jacksonVersion}" + implementation group: 'org.apache.avro', name: 'avro', version: "${avroVersion}" } checkstyle { diff --git a/native/src/main/java/io/ballerina/lib/avro/Avro.java b/native/src/main/java/io/ballerina/lib/avro/Avro.java index 9d067fd..fc65662 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Avro.java +++ b/native/src/main/java/io/ballerina/lib/avro/Avro.java @@ -18,27 +18,34 @@ package io.ballerina.lib.avro; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.avro.AvroMapper; -import com.fasterxml.jackson.dataformat.avro.AvroSchema; +import io.ballerina.lib.avro.deserialize.DeserializeFactory; +import io.ballerina.lib.avro.deserialize.Deserializer; +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.lib.avro.serialize.MessageFactory; +import io.ballerina.lib.avro.serialize.Serializer; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; import io.ballerina.runtime.api.creators.ValueCreator; -import io.ballerina.runtime.api.utils.JsonUtils; -import io.ballerina.runtime.api.utils.StringUtils; -import io.ballerina.runtime.api.utils.ValueUtils; import io.ballerina.runtime.api.values.BArray; import io.ballerina.runtime.api.values.BObject; import io.ballerina.runtime.api.values.BString; import io.ballerina.runtime.api.values.BTypedesc; import org.apache.avro.Schema; +import org.apache.avro.generic.GenericDatumReader; +import org.apache.avro.generic.GenericDatumWriter; +import org.apache.avro.io.BinaryDecoder; +import org.apache.avro.io.BinaryEncoder; +import org.apache.avro.io.DatumReader; +import org.apache.avro.io.DatumWriter; +import org.apache.avro.io.DecoderFactory; +import org.apache.avro.io.EncoderFactory; -import java.io.IOException; +import java.io.ByteArrayOutputStream; import java.util.Objects; import static io.ballerina.lib.avro.Utils.AVRO_SCHEMA; import static io.ballerina.lib.avro.Utils.DESERIALIZATION_ERROR; -import static io.ballerina.lib.avro.Utils.JSON_PROCESSING_ERROR; +import static io.ballerina.lib.avro.Utils.SERIALIZATION_ERROR; +import static io.ballerina.lib.avro.Utils.createError; public final class Avro { @@ -52,46 +59,31 @@ public static void generateSchema(BObject schemaObject, BString schema) { public static Object toAvro(BObject schemaObject, Object data) { Schema schema = (Schema) schemaObject.getNativeData(AVRO_SCHEMA); - ObjectMapper objectMapper = new ObjectMapper(); - try { - Object jsonObject = generateJsonObject(data, schema, objectMapper); - if (jsonObject == null) { - return ValueCreator.createArrayValue(new byte[]{0, 0}); - } else if (Objects.equals(schema.getType(), Schema.Type.FIXED)) { - return ValueCreator.createArrayValue(((BArray) jsonObject).getByteArray()); - } - byte[] avroBytes = (new AvroMapper()).writer(new AvroSchema(schema)).writeValueAsBytes(jsonObject); - return ValueCreator.createArrayValue(avroBytes); - } catch (JsonProcessingException e) { - return Utils.createError(JSON_PROCESSING_ERROR, e); + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + SerializeVisitor serializeVisitor = new SerializeVisitor(); + Serializer serializer = MessageFactory.createMessage(schema); + Object avroData = Objects.requireNonNull(serializer).convert(serializeVisitor, data); + DatumWriter writer = new GenericDatumWriter<>(schema); + BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(outputStream, null); + writer.write(avroData, encoder); + encoder.flush(); + return ValueCreator.createArrayValue(outputStream.toByteArray()); + } catch (Exception e) { + return Utils.createError(SERIALIZATION_ERROR, e); } } public static Object fromAvro(BObject schemaObject, BArray payload, BTypedesc typeParam) { Schema schema = (Schema) schemaObject.getNativeData(AVRO_SCHEMA); - byte[] avroBytes = payload.getByteArray(); - if (Schema.Type.FIXED.equals(schema.getType())) { - return ValueUtils.convert(ValueCreator.createArrayValue(avroBytes), typeParam.getDescribingType()); - } - JsonNode deserializedJsonString; + DatumReader datumReader = new GenericDatumReader<>(schema); + BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(payload.getBytes(), null); try { - AvroMapper mapper = new AvroMapper(); - deserializedJsonString = mapper.readerFor(Object.class).with(new AvroSchema(schema)).readTree(avroBytes); - } catch (IOException e) { - return Utils.createError(DESERIALIZATION_ERROR, e); - } - Object jsonObject = JsonUtils.parse(deserializedJsonString.toPrettyString()); - return ValueUtils.convert(jsonObject, typeParam.getDescribingType()); - } - - private static Object generateJsonObject(Object data, Schema schema, - ObjectMapper objectMapper) throws JsonProcessingException { - if (Schema.Type.NULL.equals(schema.getType()) || Schema.Type.FIXED.equals(schema.getType())) { - return data; - } else if (Schema.Type.STRING.equals(schema.getType()) || Schema.Type.ENUM.equals(schema.getType())) { - return objectMapper.readValue("\"" + data + "\"", Object.class); + Object data = datumReader.read(payload, decoder); + DeserializeVisitor deserializeVisitor = new DeserializeVisitor(); + Deserializer deserializer = DeserializeFactory.generateDeserializer(schema, typeParam.getDescribingType()); + return Objects.requireNonNull(deserializer).accept(deserializeVisitor, data); + } catch (Exception e) { + return createError(DESERIALIZATION_ERROR, e); } - Object jsonString = JsonUtils.parse(StringUtils.getJsonString(data)); - return objectMapper.readValue(jsonString.toString(), Object.class); } } diff --git a/native/src/main/java/io/ballerina/lib/avro/Utils.java b/native/src/main/java/io/ballerina/lib/avro/Utils.java index 7b7fc5b..3f2bd5a 100644 --- a/native/src/main/java/io/ballerina/lib/avro/Utils.java +++ b/native/src/main/java/io/ballerina/lib/avro/Utils.java @@ -18,8 +18,12 @@ package io.ballerina.lib.avro; +import io.ballerina.runtime.api.TypeTags; import io.ballerina.runtime.api.creators.ErrorCreator; +import io.ballerina.runtime.api.types.IntersectionType; +import io.ballerina.runtime.api.types.Type; import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; import io.ballerina.runtime.api.values.BError; import static io.ballerina.lib.avro.ModuleUtils.getModule; @@ -31,11 +35,25 @@ private Utils() { public static final String AVRO_SCHEMA = "avroSchema"; public static final String ERROR_TYPE = "Error"; - public static final String JSON_PROCESSING_ERROR = "JSON processing error"; - public static final String DESERIALIZATION_ERROR = "Unable to deserialize the byte value"; + public static final String SERIALIZATION_ERROR = "Avro serialization error"; + public static final String DESERIALIZATION_ERROR = "Avro deserialization error"; public static BError createError(String message, Throwable throwable) { BError cause = ErrorCreator.createError(throwable); return ErrorCreator.createError(getModule(), ERROR_TYPE, StringUtils.fromString(message), cause, null); } + + public static Type getMutableType(Type dataType) { + if (dataType.getTag() != TypeTags.INTERSECTION_TAG) { + return dataType; + } + IntersectionType intersectionType = (IntersectionType) dataType; + for (Type type : intersectionType.getConstituentTypes()) { + Type referredType = TypeUtils.getImpliedType(type); + if (TypeUtils.getImpliedType(intersectionType.getEffectiveType()).getTag() == referredType.getTag()) { + return referredType; + } + } + throw new IllegalStateException("Unsupported intersection type found."); + } } diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java new file mode 100644 index 0000000..417599e --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/ArrayDeserializer.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public class ArrayDeserializer extends Deserializer { + + public ArrayDeserializer(Type type, Schema schema) { + super(type, schema); + } + + @Override + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { + return visitor.visit(this, (GenericData.Array) data); + } + + @Override + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java new file mode 100644 index 0000000..42069a4 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/DeserializeFactory.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; + +public class DeserializeFactory { + + public static Deserializer generateDeserializer(Schema schema, Type type) { + return switch (schema.getType()) { + case ARRAY -> new ArrayDeserializer(type, schema); + case FIXED -> new FixedDeserializer(type, schema); + case MAP -> new MapDeserializer(schema, type); + case RECORD -> new RecordDeserializer(type, schema); + default -> new PrimitiveDeserializer(type, schema); + }; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java new file mode 100644 index 0000000..ea48aa4 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/Deserializer.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.TypeUtils; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public abstract class Deserializer { + + private final Schema schema; + private final Type type; + + public Deserializer() { + this(null, null); + } + + public Deserializer(Type type, Schema schema) { + this.schema = schema == null ? null : schema; + this.type = type == null ? null : TypeUtils.getReferredType(type); + } + + public Schema getSchema() { + return this.schema; + } + + public Type getType() { + return this.type; + } + + public abstract Object accept(DeserializeVisitor visitor, Object data) throws Exception; + public abstract Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception; +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java new file mode 100644 index 0000000..84b4d6d --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/EnumDeserializer.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public class EnumDeserializer extends Deserializer { + + public EnumDeserializer(Type type, Schema schema) { + super(type, schema); + } + + @Override + public Object accept(DeserializeVisitor visitor, Object data) { + return visitor.visit(this, (GenericData.Array) data); + } + + @Override + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java new file mode 100644 index 0000000..03cf365 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/FixedDeserializer.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public class FixedDeserializer extends Deserializer { + + public FixedDeserializer(Type type, Schema schema) { + super(type, schema); + } + + @Override + public Object accept(DeserializeVisitor visitor, Object data) { + return visitor.visit(this, data); + } + + @Override + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java new file mode 100644 index 0000000..a3c8623 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/MapDeserializer.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Map; + +public class MapDeserializer extends Deserializer { + + public MapDeserializer(Schema schema, Type type) { + super(type, schema); + } + + @Override + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { + return visitor.visit(this, (Map) data); + } + + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return null; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java new file mode 100644 index 0000000..ae2600f --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/PrimitiveDeserializer.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public class PrimitiveDeserializer extends Deserializer { + + public PrimitiveDeserializer(Type type, Schema schema) { + super(type, schema); + } + + @Override + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { + return visitor.visit(this, data); + } + + @Override + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java new file mode 100644 index 0000000..c29828d --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/RecordDeserializer.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +public class RecordDeserializer extends Deserializer { + + public RecordDeserializer(Type type, Schema schema) { + super(type, schema); + } + + @Override + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { + return visitor.visit(this, (GenericRecord) data); + } + + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java new file mode 100644 index 0000000..f4112e6 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/UnionDeserializer.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize; + +import io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor; +import io.ballerina.runtime.api.types.Type; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public class UnionDeserializer extends Deserializer { + + public UnionDeserializer(Type type, Schema schema) { + super(type, schema); + } + + @Override + public Object accept(DeserializeVisitor visitor, Object data) throws Exception { + return visitor.visit(this, (GenericData.Array) data); + } + + public Object accept(DeserializeVisitor visitor, GenericData.Array data) throws Exception { + return visitor.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java new file mode 100644 index 0000000..5bb11c2 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeArrayVisitor.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize.visitor; + +import io.ballerina.lib.avro.Utils; +import io.ballerina.lib.avro.deserialize.ArrayDeserializer; +import io.ballerina.lib.avro.deserialize.Deserializer; +import io.ballerina.runtime.api.TypeTags; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.ArrayType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public class DeserializeArrayVisitor extends DeserializeVisitor { + + public Object visit(ArrayDeserializer arrayDeserializer, GenericData.Array data) throws Exception { + Object[] objects = new Object[data.size()]; + boolean isReadOnly = arrayDeserializer.getType().getTag() == TypeTags.INTERSECTION_TAG; + Type elementType = ((ArrayType) Utils.getMutableType(arrayDeserializer.getType())).getElementType(); + int index = 0; + for (Object element : data) { + GenericData.Array dataArray = (GenericData.Array) element; + Type arrType = elementType.getTag() == TypeTags.ARRAY_TAG ? elementType : arrayDeserializer.getType(); + Schema elementSchema = arrayDeserializer.getSchema().getElementType(); + objects[index++] = visitNestedArray(new ArrayDeserializer(arrType, elementSchema), dataArray); + } + BArray arrayValue = ValueCreator + .createArrayValue(objects, (ArrayType) Utils.getMutableType(arrayDeserializer.getType())); + if (isReadOnly) { + arrayValue.freezeDirect(); + } + return arrayValue; + } + + public Object visitNestedArray(ArrayDeserializer arrayDeserializer, + GenericData.Array data) throws Exception { + Deserializer deserializer = createDeserializer(arrayDeserializer.getSchema(), arrayDeserializer.getType()); + return deserializer.accept(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java new file mode 100644 index 0000000..b03d8eb --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/DeserializeVisitor.java @@ -0,0 +1,484 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize.visitor; + +import io.ballerina.lib.avro.Utils; +import io.ballerina.lib.avro.deserialize.ArrayDeserializer; +import io.ballerina.lib.avro.deserialize.Deserializer; +import io.ballerina.lib.avro.deserialize.EnumDeserializer; +import io.ballerina.lib.avro.deserialize.FixedDeserializer; +import io.ballerina.lib.avro.deserialize.MapDeserializer; +import io.ballerina.lib.avro.deserialize.PrimitiveDeserializer; +import io.ballerina.lib.avro.deserialize.RecordDeserializer; +import io.ballerina.lib.avro.deserialize.UnionDeserializer; +import io.ballerina.runtime.api.TypeTags; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.ArrayType; +import io.ballerina.runtime.api.types.Field; +import io.ballerina.runtime.api.types.MapType; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.ReferenceType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.utils.ValueUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static io.ballerina.lib.avro.Utils.getMutableType; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processArrayField; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processBytesField; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processMapField; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processRecordField; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processStringField; +import static io.ballerina.lib.avro.deserialize.visitor.RecordUtils.processUnionField; +import static io.ballerina.runtime.api.utils.StringUtils.fromString; + +public class DeserializeVisitor implements IDeserializeVisitor { + + public static Deserializer createDeserializer(Schema schema, Type type) { + return switch (schema.getElementType().getType()) { + case UNION -> + new UnionDeserializer(type, schema); + case ARRAY -> + new ArrayDeserializer(type, schema); + case ENUM -> + new EnumDeserializer(type, schema); + case RECORD -> + new RecordDeserializer(type, schema); + case FIXED -> + new FixedDeserializer(type, schema); + default -> + new PrimitiveDeserializer(type, schema); + }; + } + + public BMap visit(RecordDeserializer recordDeserializer, GenericRecord rec) throws Exception { + Type originalType = recordDeserializer.getType(); + Type type = recordDeserializer.getType(); + Schema schema = recordDeserializer.getSchema(); + BMap avroRecord = createAvroRecord(type); + for (Schema.Field field : schema.getFields()) { + Object fieldData = rec.get(field.name()); + switch (field.schema().getType()) { + case MAP -> + processMapField(avroRecord, field, fieldData); + case ARRAY -> { + Type fieldType = ((RecordType) avroRecord.getType()).getFields().get(field.name()).getFieldType(); + processArrayField(avroRecord, field, fieldData, fieldType); + } + case BYTES -> + processBytesField(avroRecord, field, fieldData); + case RECORD -> + processRecordField(avroRecord, field, fieldData); + case STRING -> + processStringField(avroRecord, field, fieldData); + case INT -> + avroRecord.put(fromString(field.name()), Long.parseLong(fieldData.toString())); + case FLOAT -> + avroRecord.put(fromString(field.name()), Double.parseDouble(fieldData.toString())); + case UNION -> + processUnionField(type, avroRecord, field, fieldData); + default -> + avroRecord.put(fromString(field.name()), fieldData); + } + } + + if (originalType.isReadOnly()) { + avroRecord.freezeDirect(); + } + return avroRecord; + } + + public BMap visit(MapDeserializer mapDeserializer, Map data) throws Exception { + BMap avroRecord = ValueCreator.createMapValue(); + Object[] keys = data.keySet().toArray(); + Schema schema = mapDeserializer.getSchema(); + Type type = mapDeserializer.getType(); + for (Object key : keys) { + Object value = data.get(key); + Schema.Type valueType = schema.getValueType().getType(); + switch (valueType) { + case ARRAY -> + processMapArray(avroRecord, schema, + (MapType) getMutableType(type), key, (GenericData.Array) value); + case BYTES -> + avroRecord.put(StringUtils.fromString(key.toString()), + ValueCreator.createArrayValue(((ByteBuffer) value).array())); + case FIXED -> + avroRecord.put(StringUtils.fromString(key.toString()), + ValueCreator.createArrayValue(((GenericFixed) value).bytes())); + case ENUM, STRING -> + avroRecord.put(StringUtils.fromString(key.toString()), + StringUtils.fromString(value.toString())); + case RECORD -> + processMapRecord(avroRecord, schema, (MapType) getMutableType(type), + key, (GenericRecord) value); + case FLOAT -> + avroRecord.put(StringUtils.fromString(key.toString()), + Double.parseDouble(value.toString())); + case MAP -> + processMaps(avroRecord, schema, (MapType) getMutableType(type), + key, (Map) value); + default -> + avroRecord.put(StringUtils.fromString(key.toString()), value); + } + } + return (BMap) ValueUtils.convert(avroRecord, type); + } + + public Object visit(PrimitiveDeserializer primitiveDeserializer, Object data) throws Exception { + Schema schema = primitiveDeserializer.getSchema(); + Type type = primitiveDeserializer.getType(); + switch(schema.getType()) { + case ARRAY -> { + return visitPrimitiveArrays(primitiveDeserializer, (GenericData.Array) data, schema, type); + } + case STRING, ENUM -> { + return StringUtils.fromString(data.toString()); + } + case FLOAT, DOUBLE -> { + if (data instanceof Float) { + return Double.parseDouble(data.toString()); + } + return data; + } + case NULL -> { + if (data != null) { + throw new Exception("The value does not match with the null schema"); + } + return null; + } + case BYTES -> { + return ValueCreator.createArrayValue(((ByteBuffer) data).array()); + } + default -> { + return data; + } + } + } + + private Object visitPrimitiveArrays(PrimitiveDeserializer primitiveDeserializer, GenericData.Array data, + Schema schema, Type type) { + switch (schema.getElementType().getType()) { + case STRING -> { + return ValueUtils.convert(visitStringArray(data), type); + } + case INT -> { + return ValueUtils.convert(visitIntArray(data), type); + } + case LONG -> { + return ValueUtils.convert(visitLongArray(data), type); + } + case FLOAT, DOUBLE -> { + return ValueUtils.convert(visitDoubleArray(data), type); + } + case BOOLEAN -> { + return ValueUtils.convert(visitBooleanArray(data), type); + } + default -> { + return ValueUtils.convert(visitBytesArray(data, primitiveDeserializer.getType()), type); + } + } + } + + public BArray visit(UnionDeserializer unionDeserializer, GenericData.Array data) throws Exception { + Type type = unionDeserializer.getType(); + Schema schema = unionDeserializer.getSchema(); + switch (((ArrayType) type).getElementType().getTag()) { + case TypeTags.STRING_TAG -> { + return visitStringArray(data); + } + case TypeTags.FLOAT_TAG -> { + return visitDoubleArray(data); + } + case TypeTags.BOOLEAN_TAG -> { + return visitBooleanArray(data); + } + case TypeTags.INT_TAG -> { + return visitIntegerArray(data, schema); + } + case TypeTags.RECORD_TYPE_TAG -> { + return visitRecordArray(data, type, schema); + } + case TypeTags.ARRAY_TAG -> { + return visitUnionArray(data, (ArrayType) type, schema); + } + default -> { + return (BArray) data; + } + } + } + + private BArray visitRecordArray(GenericData.Array data, Type type, Schema schema) throws Exception { + RecordDeserializer recordDeserializer = new RecordDeserializer(type, schema.getElementType()); + return (BArray) recordDeserializer.accept(this, data); + } + + private BArray visitUnionArray(GenericData.Array data, ArrayType type, Schema schema) throws Exception { + Object[] objects = new Object[data.size()]; + Type elementType = type.getElementType(); + ArrayDeserializer arrayDeserializer = new ArrayDeserializer(elementType, schema.getElementType()); + int index = 0; + for (Object currentData : data) { + Object deserializedObject = arrayDeserializer.accept(this, (GenericData.Array) currentData); + objects[index++] = deserializedObject; + } + return ValueCreator.createArrayValue(objects, type); + } + + public BArray visit(RecordDeserializer recordDeserializer, GenericData.Array data) throws Exception { + List recordList = new ArrayList<>(); + boolean isReadOnly = recordDeserializer.getType().getTag() == TypeTags.INTERSECTION_TAG; + Type type = Utils.getMutableType(recordDeserializer.getType()); + Schema schema = recordDeserializer.getSchema(); + switch (type.getTag()) { + case TypeTags.ARRAY_TAG -> { + for (Object datum : data) { + Type fieldType = ((ArrayType) type).getElementType(); + RecordDeserializer recordDes = new RecordDeserializer(fieldType, schema.getElementType()); + recordList.add(recordDes.accept(this, datum)); + } + } + case TypeTags.TYPE_REFERENCED_TYPE_TAG -> { + for (Object datum : data) { + Type fieldType = ((ReferenceType) type).getReferredType(); + RecordDeserializer recordDes = new RecordDeserializer(fieldType, schema.getElementType()); + recordList.add(recordDes.accept(this, (GenericRecord) datum)); + } + } + } + BArray arrayValue = ValueCreator.createArrayValue(recordList.toArray(new Object[data.size()]), + (ArrayType) type); + if (isReadOnly) { + arrayValue.freezeDirect(); + } + return arrayValue; + } + + private BMap createAvroRecord(Type type) { + return ValueCreator.createRecordValue((RecordType) getMutableType(type)); + } + + private void processMaps(BMap avroRecord, Schema schema, + MapType type, Object key, Map value) throws Exception { + Schema fieldSchema = schema.getValueType(); + Type fieldType = type.getConstrainedType(); + MapDeserializer mapDes = new MapDeserializer(fieldSchema, fieldType); + Object fieldValue = mapDes.accept(this, value); + avroRecord.put(fromString(key.toString()), fieldValue); + } + + private void processMapRecord(BMap avroRecord, Schema schema, + MapType type, Object key, GenericRecord value) throws Exception { + Type fieldType = type.getConstrainedType(); + RecordDeserializer recordDes = new RecordDeserializer(fieldType, schema.getValueType()); + Object fieldValue = recordDes.accept(this, value); + avroRecord.put(fromString(key.toString()), fieldValue); + } + + private void processMapArray(BMap avroRecord, Schema schema, + MapType type, Object key, GenericData.Array value) throws Exception { + Type fieldType = type.getConstrainedType(); + ArrayDeserializer arrayDeserializer = new ArrayDeserializer(fieldType, schema.getValueType()); + Object fieldValue = visit(arrayDeserializer, value); + avroRecord.put(fromString(key.toString()), fieldValue); + } + + public Object visit(ArrayDeserializer arrayDeserializer, GenericData.Array data) throws Exception { + Deserializer deserializer = createDeserializer(arrayDeserializer.getSchema(), arrayDeserializer.getType()); + return deserializer.accept(new DeserializeArrayVisitor(), data); + } + + public BArray visit(EnumDeserializer enumDeserializer, GenericData.Array data) { + Object[] enums = new Object[data.size()]; + for (int i = 0; i < data.size(); i++) { + enums[i] = visitString(data.get(i)); + } + return ValueCreator.createArrayValue(enums, (ArrayType) enumDeserializer.getType()); + } + + public Object visit(FixedDeserializer fixedDeserializer, Object data) { + if (fixedDeserializer.getSchema().getType().equals(Schema.Type.ARRAY)) { + GenericData.Array array = (GenericData.Array) data; + Type type = fixedDeserializer.getType(); + List values = new ArrayList<>(); + for (Object datum : array) { + values.add(visitFixed(datum)); + } + return ValueCreator.createArrayValue(values.toArray(new BArray[array.size()]), (ArrayType) type); + } else { + return visitFixed(data); + } + } + + public BArray visit(FixedDeserializer fixedDeserializer, GenericData.Array data) { + Type type = fixedDeserializer.getType(); + List values = new ArrayList<>(); + for (Object datum : data) { + values.add(visitFixed(datum)); + } + return ValueCreator.createArrayValue(values.toArray(new BArray[data.size()]), (ArrayType) type); + } + + private static BArray visitIntegerArray(GenericData.Array data, Schema schema) { + for (Schema schemaInstance : schema.getElementType().getTypes()) { + if (schemaInstance.getType().equals(Schema.Type.INT)) { + return visitIntArray(data); + } + } + return visitLongArray(data); + } + + private BArray visitBytesArray(GenericData.Array data, Type type) { + List values = new ArrayList<>(); + for (Object datum : data) { + values.add(ValueCreator.createArrayValue(((ByteBuffer) datum).array())); + } + return ValueCreator.createArrayValue(values.toArray(new BArray[data.size()]), (ArrayType) type); + } + + private static BArray visitBooleanArray(GenericData.Array data) { + boolean[] booleanArray = new boolean[data.size()]; + int index = 0; + for (Object datum : data) { + booleanArray[index++] = (boolean) datum; + } + return ValueCreator.createArrayValue(booleanArray); + } + + private BArray visitDoubleArray(GenericData.Array data) { + List doubleList = new ArrayList<>(); + for (Object datum : data) { + doubleList.add(visitDouble(datum)); + } + double[] doubleArray = doubleList.stream().mapToDouble(Double::doubleValue).toArray(); + return ValueCreator.createArrayValue(doubleArray); + } + + private static BArray visitLongArray(GenericData.Array data) { + List longList = new ArrayList<>(); + for (Object datum : data) { + longList.add((Long) datum); + } + long[] longArray = longList.stream().mapToLong(Long::longValue).toArray(); + return ValueCreator.createArrayValue(longArray); + } + + private static BArray visitIntArray(GenericData.Array data) { + List longList = new ArrayList<>(); + for (Object datum : data) { + longList.add(((Integer) datum).longValue()); + } + long[] longArray = longList.stream().mapToLong(Long::longValue).toArray(); + return ValueCreator.createArrayValue(longArray); + } + + private BArray visitStringArray(GenericData.Array data) { + BString[] stringArray = new BString[data.size()]; + for (int i = 0; i < data.size(); i++) { + stringArray[i] = visitString(data.get(i)); + } + return ValueCreator.createArrayValue(stringArray); + } + + + public double visitDouble(Object data) { + if (data instanceof Float) { + return Double.parseDouble(data.toString()); + } + return (double) data; + } + + public BArray visitFixed(Object data) { + GenericData.Fixed fixed = (GenericData.Fixed) data; + return ValueCreator.createArrayValue(fixed.bytes()); + } + + public BString visitString(Object data) { + return fromString(data.toString()); + } + + public static Type extractMapType(Type type) throws Exception { + Type mapType = type; + if (type.getTag() != TypeTags.RECORD_TYPE_TAG) { + throw new Exception("Type is not a record type."); + } + for (Map.Entry entry : ((RecordType) type).getFields().entrySet()) { + Field fieldValue = entry.getValue(); + if (fieldValue != null) { + Type fieldType = fieldValue.getFieldType(); + switch (fieldType.getTag()) { + case TypeTags.MAP_TAG -> + mapType = fieldType; + case TypeTags.INTERSECTION_TAG -> { + Type referredType = getMutableType(fieldType); + if (referredType.getTag() == TypeTags.MAP_TAG) { + mapType = referredType; + } + } + default -> { + Type referType = TypeUtils.getReferredType(fieldType); + if (referType.getTag() == TypeTags.MAP_TAG) { + mapType = referType; + } + } + } + } + } + return mapType; + } + + public static RecordType extractRecordType(RecordType type) { + Map fieldsMap = type.getFields(); + RecordType recType = type; + for (Map.Entry entry : fieldsMap.entrySet()) { + Field fieldValue = entry.getValue(); + if (fieldValue != null) { + Type fieldType = fieldValue.getFieldType(); + switch (fieldType.getTag()) { + case TypeTags.RECORD_TYPE_TAG -> + recType = (RecordType) fieldType; + case TypeTags.INTERSECTION_TAG -> { + if (getMutableType(fieldType).getTag() == TypeTags.RECORD_TYPE_TAG) { + recType = (RecordType) getMutableType(fieldType); + } + } + default -> { + Type referredType = TypeUtils.getReferredType(fieldType); + if (referredType.getTag() == TypeTags.RECORD_TYPE_TAG) { + recType = (RecordType) referredType; + } + } + } + } + } + return recType; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/IDeserializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/IDeserializeVisitor.java new file mode 100644 index 0000000..7bac24c --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/IDeserializeVisitor.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize.visitor; + +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BString; + +public interface IDeserializeVisitor { + + public double visitDouble(Object data); + public BString visitString(Object data); + BArray visitFixed(Object data); +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java new file mode 100644 index 0000000..42d0411 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/RecordUtils.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize.visitor; + +import io.ballerina.lib.avro.deserialize.ArrayDeserializer; +import io.ballerina.lib.avro.deserialize.MapDeserializer; +import io.ballerina.lib.avro.deserialize.PrimitiveDeserializer; +import io.ballerina.lib.avro.deserialize.RecordDeserializer; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.RecordType; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.nio.ByteBuffer; + +import static io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor.extractMapType; +import static io.ballerina.lib.avro.deserialize.visitor.DeserializeVisitor.extractRecordType; +import static io.ballerina.lib.avro.deserialize.visitor.UnionRecordUtils.visitUnionRecords; +import static io.ballerina.runtime.api.utils.StringUtils.fromString; + +public class RecordUtils { + + public static void processMapField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + Type mapType = extractMapType(avroRecord.getType()); + MapDeserializer mapDeserializer = new MapDeserializer(field.schema(), mapType); + Object fieldValue = mapDeserializer.accept(new DeserializeVisitor(), fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + public static void processArrayField(BMap avroRecord, + Schema.Field field, Object fieldData, Type type) throws Exception { + ArrayDeserializer arrayDes = new ArrayDeserializer(type, field.schema()); + Object fieldValue = arrayDes.accept(new DeserializeVisitor(), (GenericData.Array) fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + public static void processBytesField(BMap avroRecord, Schema.Field field, Object fieldData) { + ByteBuffer byteBuffer = (ByteBuffer) fieldData; + Object fieldValue = ValueCreator.createArrayValue(byteBuffer.array()); + avroRecord.put(fromString(field.name()), fieldValue); + } + + public static void processRecordField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + Type recType = extractRecordType((RecordType) avroRecord.getType()); + RecordDeserializer recordDes = new RecordDeserializer(recType, field.schema()); + Object fieldValue = recordDes.accept(new DeserializeVisitor(), fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + public static void processStringField(BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + PrimitiveDeserializer stringDes = new PrimitiveDeserializer(null, field.schema()); + Object fieldValue = stringDes.accept(new DeserializeVisitor(), fieldData); + avroRecord.put(fromString(field.name()), fieldValue); + } + + public static void processUnionField(Type type, BMap avroRecord, + Schema.Field field, Object fieldData) throws Exception { + visitUnionRecords(type, avroRecord, field, fieldData); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java new file mode 100644 index 0000000..b56484f --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/deserialize/visitor/UnionRecordUtils.java @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.deserialize.visitor; + +import io.ballerina.lib.avro.deserialize.RecordDeserializer; +import io.ballerina.runtime.api.creators.ValueCreator; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import io.ballerina.runtime.api.values.BString; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericEnumSymbol; +import org.apache.avro.generic.GenericFixed; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.util.Utf8; + +import java.nio.ByteBuffer; +import java.util.Map; + +public class UnionRecordUtils { + + public static void visitUnionRecords(Type type, BMap ballerinaRecord, + Schema.Field field, Object fieldData) throws Exception { + for (Schema schemaType : field.schema().getTypes()) { + if (fieldData == null) { + ballerinaRecord.put(StringUtils.fromString(field.name()), null); + break; + } + switch (schemaType.getType()) { + case BYTES: + handleBytesField(field, fieldData, ballerinaRecord); + break; + case FIXED: + handleFixedField(field, fieldData, ballerinaRecord); + break; + case ARRAY: + handleArrayField(field, fieldData, ballerinaRecord, schemaType); + break; + case MAP: + handleMapField(field, fieldData, ballerinaRecord); + break; + case RECORD: + handleRecordField(type, field, fieldData, ballerinaRecord, schemaType); + break; + case STRING: + handleStringField(field, fieldData, ballerinaRecord); + break; + case INT, LONG: + handleIntegerField(field, fieldData, ballerinaRecord); + break; + case FLOAT, DOUBLE: + handleFloatField(field, fieldData, ballerinaRecord); + break; + case ENUM: + handleEnumField(field, fieldData, ballerinaRecord); + break; + default: + handleDefaultField(field, fieldData, ballerinaRecord); + } + } + } + + private static void handleDefaultField(Schema.Field field, Object fieldData, + BMap ballerinaRecord) { + if (fieldData instanceof Boolean) { + ballerinaRecord.put(StringUtils.fromString(field.name()), fieldData); + } + } + + private static void handleEnumField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof GenericEnumSymbol) { + ballerinaRecord.put(StringUtils.fromString(field.name()), StringUtils.fromString(fieldData.toString())); + } + } + + private static void handleFloatField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof Double) { + ballerinaRecord.put(StringUtils.fromString(field.name()), fieldData); + } else { + ballerinaRecord.put(StringUtils.fromString(field.name()), Double.parseDouble(fieldData.toString())); + } + } + + private static void handleIntegerField(Schema.Field field, Object fieldData, + BMap ballerinaRecord) { + if (fieldData instanceof Integer || fieldData instanceof Long) { + ballerinaRecord.put(StringUtils.fromString(field.name()), ((Number) fieldData).longValue()); + } + } + + private static void handleStringField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof Utf8) { + ballerinaRecord.put(StringUtils.fromString(field.name()), StringUtils.fromString(fieldData.toString())); + } + } + + public static void handleRecordField(Type type, Schema.Field field, Object fieldData, + BMap ballerinaRecord, Schema schemaType) throws Exception { + if (fieldData instanceof GenericRecord) { + RecordDeserializer recordDes = new RecordDeserializer(type, schemaType); + Object fieldValue = recordDes.accept(new DeserializeVisitor(), (GenericRecord) fieldData); + ballerinaRecord.put(StringUtils.fromString(field.name()), fieldValue); + } + } + + private static void handleMapField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof Map) { + BMap avroMap = ValueCreator.createMapValue(); + Object[] keys = ((Map) fieldData).keySet().toArray(); + for (Object key : keys) { + avroMap.put(StringUtils.fromString(key.toString()), + ((Map) fieldData).get(key)); + } + ballerinaRecord.put(StringUtils.fromString(field.name()), avroMap); + } + } + + private static void handleBytesField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof ByteBuffer) { + BArray byteArray = ValueCreator.createArrayValue(((ByteBuffer) fieldData).array()); + ballerinaRecord.put(StringUtils.fromString(field.name()), byteArray); + } + } + + private static void handleFixedField(Schema.Field field, Object fieldData, BMap ballerinaRecord) { + if (fieldData instanceof GenericFixed) { + BArray byteArray = ValueCreator.createArrayValue(((GenericData.Fixed) fieldData).bytes()); + ballerinaRecord.put(StringUtils.fromString(field.name()), byteArray); + } + } + + private static void handleArrayField(Schema.Field field, Object fieldData, + BMap ballerinaRecord, Schema schemaType) { + if (fieldData instanceof GenericData.Array) { + Object[] objectArray = ((GenericData.Array) fieldData).toArray(); + if (schemaType.getElementType().getType().equals(Schema.Type.STRING) + || schemaType.getElementType().getType().equals(Schema.Type.ENUM)) { + BString[] stringArray = new BString[objectArray.length]; + BArray ballerinaArray = ValueCreator.createArrayValue(stringArray); + int i = 0; + for (Object obj : objectArray) { + stringArray[i] = StringUtils.fromString(obj.toString()); + i++; + } + ballerinaRecord.put(StringUtils.fromString(field.name()), ballerinaArray); + } else { + ballerinaRecord.put(StringUtils.fromString(field.name()), fieldData); + } + } + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java new file mode 100644 index 0000000..7b51923 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/ArraySerializer.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; + +public class ArraySerializer extends Serializer { + + public ArraySerializer(Schema schema) { + super(schema); + } + + @Override + public Object convert(SerializeVisitor serializeVisitor, Object data) { + return serializeVisitor.visit(this, (BArray) data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java new file mode 100644 index 0000000..c24905b --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/ByteSerializer.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BArray; + +import java.nio.ByteBuffer; + +public class ByteSerializer extends Serializer { + + @Override + public Object convert(SerializeVisitor serializeVisitor, Object data) { + return ByteBuffer.wrap(((BArray) data).getByteArray()); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java new file mode 100644 index 0000000..9f50b5a --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/EnumSerializer.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import org.apache.avro.Schema; + +public class EnumSerializer extends Serializer { + + public EnumSerializer(Schema schema) { + super(schema); + } + + @Override + public Object convert(SerializeVisitor serializeVisitor, Object data) { + return serializeVisitor.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java new file mode 100644 index 0000000..68b9317 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/FixedSerializer.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import org.apache.avro.Schema; + +public class FixedSerializer extends Serializer { + + public FixedSerializer(Schema schema) { + super(schema); + } + + @Override + public Object convert(SerializeVisitor serializeVisitor, Object data) { + return serializeVisitor.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java new file mode 100644 index 0000000..b7be25b --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/MapSerializer.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.Schema; + +public class MapSerializer extends Serializer { + + public MapSerializer(Schema schema) { + super(schema); + } + + @Override + public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { + return serializeVisitor.visit(this, (BMap) data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java new file mode 100644 index 0000000..ee5bc68 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/MessageFactory.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import org.apache.avro.Schema; + +public class MessageFactory { + + public static Serializer createMessage(Schema schema) { + return switch (schema.getType()) { + case ARRAY -> new ArraySerializer(schema); + case FIXED -> new FixedSerializer(schema); + case ENUM -> new EnumSerializer(schema); + case MAP -> new MapSerializer(schema); + case RECORD -> new RecordSerializer(schema); + case BYTES -> new ByteSerializer(); + default -> new PrimitiveSerializer(schema); + }; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java new file mode 100644 index 0000000..e6281ef --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/NullSerializer.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; + +public class NullSerializer extends Serializer { + + @Override + public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { + if (data != null) { + throw new Exception("The value does not match with the null schema"); + } + return null; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveSerializer.java new file mode 100644 index 0000000..080c1d4 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/PrimitiveSerializer.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import org.apache.avro.Schema; + +public class PrimitiveSerializer extends Serializer { + + public PrimitiveSerializer(Schema schema) { + super(schema); + } + + @Override + public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { + return serializeVisitor.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java new file mode 100644 index 0000000..bb9b7fb --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/RecordSerializer.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.Schema; + +public class RecordSerializer extends Serializer { + + public RecordSerializer(Schema schema) { + super(schema); + } + + @Override + public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { + return serializeVisitor.visit(this, (BMap) data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java new file mode 100644 index 0000000..ef0f1f2 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/Serializer.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.TypeUtils; +import org.apache.avro.Schema; + +public abstract class Serializer { + + private final Schema schema; + private final Type type; + + public Serializer() { + this.type = null; + this.schema = null; + } + + public Serializer(Schema schema) { + this.type = null; + this.schema = schema; + } + + public Serializer(Schema schema, Type type) { + this.type = TypeUtils.getImpliedType(type); + this.schema = schema; + } + + public Schema getSchema() { + return this.schema; + } + + public Type getType() { + return this.type; + } + + public abstract Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception; +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java b/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java new file mode 100644 index 0000000..a4ef7f3 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/UnionSerializer.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize; + +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import org.apache.avro.Schema; + +public class UnionSerializer extends Serializer { + + public UnionSerializer(Schema schema) { + super(schema); + } + + @Override + public Object convert(SerializeVisitor serializeVisitor, Object data) throws Exception { + return serializeVisitor.visit(this, data); + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java new file mode 100644 index 0000000..ffe79de --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/ISerializeVisitor.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor; + +import io.ballerina.lib.avro.serialize.ArraySerializer; +import io.ballerina.lib.avro.serialize.EnumSerializer; +import io.ballerina.lib.avro.serialize.FixedSerializer; +import io.ballerina.lib.avro.serialize.PrimitiveSerializer; +import io.ballerina.lib.avro.serialize.RecordSerializer; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +public interface ISerializeVisitor { + + GenericRecord visit(RecordSerializer recordSerializer, BMap data) throws Exception; + GenericData.Array visit(ArraySerializer arraySerializer, BArray data); + Object visit(EnumSerializer enumSerializer, Object data); + GenericData.Fixed visit(FixedSerializer fixedSerializer, Object data); + Object visit(PrimitiveSerializer primitiveSerializer, Object data) throws Exception; +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java new file mode 100644 index 0000000..4c52977 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/SerializeVisitor.java @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor; + +import io.ballerina.lib.avro.serialize.ArraySerializer; +import io.ballerina.lib.avro.serialize.EnumSerializer; +import io.ballerina.lib.avro.serialize.FixedSerializer; +import io.ballerina.lib.avro.serialize.MapSerializer; +import io.ballerina.lib.avro.serialize.PrimitiveSerializer; +import io.ballerina.lib.avro.serialize.RecordSerializer; +import io.ballerina.lib.avro.serialize.Serializer; +import io.ballerina.lib.avro.serialize.UnionSerializer; +import io.ballerina.lib.avro.serialize.visitor.array.ArrayVisitorFactory; +import io.ballerina.lib.avro.serialize.visitor.array.IArrayVisitor; +import io.ballerina.runtime.api.TypeTags; +import io.ballerina.runtime.api.types.Type; +import io.ballerina.runtime.api.utils.StringUtils; +import io.ballerina.runtime.api.utils.TypeUtils; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class SerializeVisitor implements ISerializeVisitor { + + public Serializer createSerializer(Schema schema) { + return switch (schema.getValueType().getType()) { + case INT, LONG, FLOAT, DOUBLE, BOOLEAN, STRING, BYTES -> + new PrimitiveSerializer(schema.getValueType()); + case RECORD -> + new RecordSerializer(schema.getValueType()); + case MAP -> + new MapSerializer(schema.getValueType()); + case ARRAY -> + new ArraySerializer(schema.getValueType()); + case ENUM -> + new EnumSerializer(schema.getValueType()); + case FIXED -> + new FixedSerializer(schema.getValueType()); + default -> + throw new IllegalArgumentException("Unsupported schema type: " + schema.getValueType().getType()); + }; + } + + @Override + public GenericRecord visit(RecordSerializer recordSerializer, BMap data) throws Exception { + GenericRecord genericRecord = new GenericData.Record(recordSerializer.getSchema()); + for (Schema.Field field : recordSerializer.getSchema().getFields()) { + Object fieldData = data.get(StringUtils.fromString(field.name())); + genericRecord.put(field.name(), serializeField(field.schema(), fieldData)); + } + return genericRecord; + } + + private Object serializeField(Schema schema, Object fieldData) throws Exception { + Schema.Type type = schema.getType(); + return switch (type) { + case RECORD -> + new RecordSerializer(schema).convert(this, fieldData); + case MAP -> + new MapSerializer(schema).convert(this, fieldData); + case ARRAY -> + new ArraySerializer(schema).convert(this, fieldData); + case ENUM -> + new EnumSerializer(schema).convert(this, fieldData); + case UNION -> + new UnionSerializer(schema).convert(this, fieldData); + default -> + new PrimitiveSerializer(schema).convert(this, fieldData); + }; + } + + @Override + public Object visit(PrimitiveSerializer primitiveSerializer, Object data) throws Exception { + switch (primitiveSerializer.getSchema().getType()) { + case INT -> { + return ((Long) data).intValue(); + } + case FLOAT -> { + return ((Double) data).floatValue(); + } + case BYTES -> { + ByteBuffer byteBuffer = ByteBuffer.allocate(((BArray) data).getByteArray().length); + byteBuffer.put(((BArray) data).getByteArray()); + byteBuffer.position(0); + return byteBuffer; + } + case STRING -> { + return data.toString(); + } + case NULL -> { + if (data != null) { + throw new Exception("The value does not match with the null schema"); + } + return null; + } + default -> { + return data; + } + } + } + + public Map visit(MapSerializer mapSerializer, BMap data) throws Exception { + Map avroMap = new HashMap<>(); + Schema schema = mapSerializer.getSchema(); + if (schema.getType().equals(Schema.Type.UNION)) { + for (Schema fieldSchema: schema.getTypes()) { + if (fieldSchema.getType().equals(Schema.Type.MAP)) { + schema = fieldSchema; + } + } + } + for (Object value : data.getKeys()) { + Serializer serializer = createSerializer(schema); + avroMap.put(value.toString(), serializer.convert(this, data.get(value))); + } + return avroMap; + } + + @Override + public Object visit(EnumSerializer enumSerializer, Object data) { + return new GenericData.EnumSymbol(enumSerializer.getSchema(), data); + } + + @Override + public GenericData.Fixed visit(FixedSerializer fixedSerializer, Object data) { + return new GenericData.Fixed(fixedSerializer.getSchema(), ((BArray) data).getByteArray()); + } + + public GenericData.Array visit(ArraySerializer arraySerializer, BArray data) { + GenericData.Array array = new GenericData.Array<>(data.size(), arraySerializer.getSchema()); + IArrayVisitor visitor = ArrayVisitorFactory.createVisitor(arraySerializer.getSchema()); + return Objects.requireNonNull(visitor).visit(data, arraySerializer.getSchema(), array); + } + + public Object visit(UnionSerializer unionSerializer, Object data) throws Exception { + Schema fieldSchema = unionSerializer.getSchema(); + Type typeName = TypeUtils.getType(data); + switch (typeName.getTag()) { + case TypeTags.STRING_TAG -> { + return visitUnionStrings(data, fieldSchema); + } + case TypeTags.ARRAY_TAG -> { + return visitUnionArrays(data, fieldSchema); + } + case TypeTags.MAP_TAG -> { + return new MapSerializer(fieldSchema).convert(this, data); + } + case TypeTags.RECORD_TYPE_TAG -> { + Schema schema = getRecordSchema(Schema.Type.RECORD, fieldSchema.getTypes()); + return new RecordSerializer(schema).convert(this, data); + } + case TypeTags.INT_TAG -> { + return visitUnionIntegers(data, fieldSchema); + } + case TypeTags.FLOAT_TAG -> { + return visitUnionFloats(data, fieldSchema); + } + default -> { + return data; + } + } + } + + private Object visitUnionFloats(Object data, Schema fieldSchema) { + return fieldSchema.getTypes().stream() + .filter(schema -> schema.getType().equals(Schema.Type.FLOAT)) + .findFirst() + .map(schema -> { + try { + return new PrimitiveSerializer(schema).convert(this, data); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .orElse(data); + } + + private Object visitUnionIntegers(Object data, Schema fieldSchema) { + return fieldSchema.getTypes().stream() + .filter(schema -> schema.getType().equals(Schema.Type.INT)) + .findFirst() + .map(schema -> { + try { + return new PrimitiveSerializer(schema).convert(this, data); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .orElse(data); + } + + private Object visitUnionStrings(Object data, Schema fieldSchema) throws Exception { + return fieldSchema.getTypes().stream() + .filter(type -> type.getType().equals(Schema.Type.ENUM)) + .findFirst() + .map(type -> visit(new EnumSerializer(type), data)) + .orElse(visit(new PrimitiveSerializer(fieldSchema), data.toString())); + } + + private Object visitUnionArrays(Object data, Schema fieldSchema) throws Exception { + for (Schema schema : fieldSchema.getTypes()) { + switch (schema.getType()) { + case BYTES -> { + return new PrimitiveSerializer(schema).convert(this, data); + } + case FIXED -> { + return new FixedSerializer(schema).convert(this, data); + } + case ARRAY -> { + return new ArraySerializer(schema).convert(this, data); + } + } + } + return new ArraySerializer(fieldSchema).convert(this, data); + } + + public static Schema getRecordSchema(Schema.Type givenType, List schemas) { + for (Schema schema: schemas) { + if (schema.getType().equals(Schema.Type.UNION)) { + getRecordSchema(givenType, schema.getTypes()); + } else if (schema.getType().equals(givenType)) { + return schema; + } + } + return null; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java new file mode 100644 index 0000000..db6aeaf --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitor.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.lib.avro.serialize.ArraySerializer; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Arrays; +import java.util.Objects; + +public class ArrayVisitor implements IArrayVisitor { + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(value -> { + try { + array.add(new SerializeVisitor().visit(new ArraySerializer(schema.getElementType()), + (BArray) value)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java new file mode 100644 index 0000000..14045b4 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/ArrayVisitorFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor.array; + +import org.apache.avro.Schema; + +public class ArrayVisitorFactory { + public static IArrayVisitor createVisitor(Schema schema) { + switch (schema.getElementType().getType()) { + case NULL: + return null; + case ARRAY: + return new ArrayVisitor(); + case ENUM: + return new EnumArrayVisitor(); + case UNION: + return new UnionArrayVisitor(); + case FIXED: + return new FixedArrayVisitor(); + case RECORD: + return new RecordArrayVisitor(); + case MAP: + return new MapArrayVisitor(); + default: + return new PrimitiveArrayVisitor(); + } + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java new file mode 100644 index 0000000..9076336 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/EnumArrayVisitor.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Arrays; +import java.util.Objects; + +public class EnumArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Arrays.stream((data.getValues() == null) ? data.getStringArray() : data.getValues()) + .filter(Objects::nonNull) + .forEach(value -> { + try { + array.add(new GenericData.EnumSymbol(schema.getElementType(), value)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java new file mode 100644 index 0000000..fec45c6 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/FixedArrayVisitor.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericFixed; + +import java.util.Arrays; +import java.util.Objects; + +public class FixedArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(bytes -> { + GenericFixed genericFixed = new GenericData.Fixed(schema, ((BArray) bytes).getByteArray()); + array.add(genericFixed); + }); + return array; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java new file mode 100644 index 0000000..e126a83 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/IArrayVisitor.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +public interface IArrayVisitor { + GenericData.Array visit(BArray data, Schema schema, GenericData.Array array); +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java new file mode 100644 index 0000000..6c169db --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/MapArrayVisitor.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.lib.avro.serialize.MapSerializer; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Arrays; +import java.util.Objects; + +public class MapArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(record -> { + try { + array.add(new SerializeVisitor().visit(new MapSerializer(schema.getElementType()), + (BMap) record)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java new file mode 100644 index 0000000..c1dc886 --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/PrimitiveArrayVisitor.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Objects; + +public class PrimitiveArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Schema.Type type = schema.getType().equals(Schema.Type.ARRAY) + ? schema.getElementType().getType() + : schema.getType(); + + switch (type) { + case STRING -> + array.addAll(Arrays.asList(data.getStringArray())); + case INT -> { + for (long obj: data.getIntArray()) { + array.add(((Long) obj).intValue()); + } + } + case LONG -> { + for (Object obj: data.getIntArray()) { + array.add(obj); + } + } + case FLOAT -> { + for (Double obj: data.getFloatArray()) { + array.add(obj.floatValue()); + } + } + case DOUBLE -> { + for (Object obj: data.getFloatArray()) { + array.add(obj); + } + } + case BOOLEAN -> { + for (Object obj: data.getBooleanArray()) { + array.add(obj); + } + } + default -> visitBytes(data, array); + } + return array; + } + + + public static GenericData.Array visitBytes(BArray data, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(bytes -> { + ByteBuffer byteBuffer = ByteBuffer.allocate(((BArray) bytes).getByteArray().length); + byteBuffer.put(((BArray) bytes).getByteArray()); + byteBuffer.position(0); + array.add(byteBuffer); + }); + return array; + } +} + diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java new file mode 100644 index 0000000..5e4a71a --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/RecordArrayVisitor.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.lib.avro.serialize.RecordSerializer; +import io.ballerina.lib.avro.serialize.visitor.SerializeVisitor; +import io.ballerina.runtime.api.values.BArray; +import io.ballerina.runtime.api.values.BMap; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Arrays; +import java.util.Objects; + +public class RecordArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Arrays.stream(data.getValues()) + .filter(Objects::nonNull) + .forEach(record -> { + try { + array.add(new SerializeVisitor() + .visit(new RecordSerializer(schema.getElementType()), (BMap) record)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return array; + } +} diff --git a/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java new file mode 100644 index 0000000..7d7e48c --- /dev/null +++ b/native/src/main/java/io/ballerina/lib/avro/serialize/visitor/array/UnionArrayVisitor.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com) + * + * WSO2 LLC. licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.ballerina.lib.avro.serialize.visitor.array; + +import io.ballerina.runtime.api.values.BArray; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; + +import java.util.Map; + +import static java.util.Map.entry; + +public class UnionArrayVisitor implements IArrayVisitor { + @Override + public GenericData.Array visit(BArray data, Schema schema, GenericData.Array array) { + Map visitorMap = Map.ofEntries( + entry(Schema.Type.ARRAY, new ArrayVisitor()), + entry(Schema.Type.MAP, new MapArrayVisitor()), + entry(Schema.Type.RECORD, new RecordArrayVisitor()), + entry(Schema.Type.FIXED, new FixedArrayVisitor()), + entry(Schema.Type.BOOLEAN, new PrimitiveArrayVisitor()), + entry(Schema.Type.STRING, new PrimitiveArrayVisitor()), + entry(Schema.Type.INT, new PrimitiveArrayVisitor()), + entry(Schema.Type.LONG, new PrimitiveArrayVisitor()), + entry(Schema.Type.DOUBLE, new PrimitiveArrayVisitor()), + entry(Schema.Type.BYTES, new PrimitiveArrayVisitor()), + entry(Schema.Type.FLOAT, new PrimitiveArrayVisitor()) + ); + + Schema elementType = schema.getElementType(); + for (Schema schema1 : elementType.getTypes()) { + IArrayVisitor visitor = visitorMap.get(schema1.getType()); + if (visitor != null) { + return visitor.visit(data, schema1, array); + } + } + return null; + } +} diff --git a/native/src/main/java/module-info.java b/native/src/main/java/module-info.java index 6b198c0..11b1546 100644 --- a/native/src/main/java/module-info.java +++ b/native/src/main/java/module-info.java @@ -20,6 +20,5 @@ requires io.ballerina.runtime; requires io.ballerina.lang; requires com.fasterxml.jackson.databind; - requires com.fasterxml.jackson.dataformat.avro; - requires avro; + requires org.apache.avro; }