From 63b104c70d6a493a124cd8fda733020af69b33f3 Mon Sep 17 00:00:00 2001 From: zhangdonghao Date: Sat, 2 Nov 2024 22:33:02 +0800 Subject: [PATCH 1/3] [Feature][core] support arrow transfers data to SeatunnelRow in arrow format --- pom.xml | 12 + seatunnel-common/pom.xml | 7 + .../arrow/converter/BigIntConverter.java | 33 + .../source/arrow/converter/BitConverter.java | 33 + .../source/arrow/converter/Converter.java | 35 + .../arrow/converter/DateDayConverter.java | 33 + .../arrow/converter/DateMilliConverter.java | 33 + .../arrow/converter/Decimal256Converter.java | 33 + .../arrow/converter/DecimalConverter.java | 33 + .../arrow/converter/DenseUnionConverter.java | 33 + .../arrow/converter/DurationConverter.java | 33 + .../converter/ExtensionTypeConverter.java | 33 + .../converter/FixedSizeBinaryConverter.java | 33 + .../converter/FixedSizeListConverter.java | 64 ++ .../arrow/converter/Float4Converter.java | 33 + .../arrow/converter/Float8Converter.java | 33 + .../source/arrow/converter/IntConverter.java | 33 + .../arrow/converter/IntervalDayConverter.java | 33 + .../IntervalMonthDayNanoConverter.java | 33 + .../converter/IntervalYearConverter.java | 33 + .../arrow/converter/LargeListConverter.java | 64 ++ .../converter/LargeVarBinaryConverter.java | 33 + .../converter/LargeVarCharConverter.java | 33 + .../source/arrow/converter/ListConverter.java | 63 ++ .../source/arrow/converter/MapConverter.java | 67 ++ .../source/arrow/converter/NullConverter.java | 33 + .../arrow/converter/SmallIntConverter.java | 34 + .../arrow/converter/StructConverter.java | 33 + .../arrow/converter/TimeMicroConverter.java | 33 + .../arrow/converter/TimeMilliConverter.java | 33 + .../arrow/converter/TimeNanoConverter.java | 33 + .../arrow/converter/TimeSecConverter.java | 33 + .../converter/TimeStampMicroConverter.java | 44 ++ .../converter/TimeStampMicroTZConverter.java | 33 + .../converter/TimeStampMilliConverter.java | 44 ++ .../converter/TimeStampMilliTZConverter.java | 33 + .../converter/TimeStampNanoConverter.java | 44 ++ .../converter/TimeStampNanoTZConverter.java | 33 + .../converter/TimeStampSecConverter.java | 33 + .../converter/TimeStampSecTZConverter.java | 33 + .../arrow/converter/TinyIntConverter.java | 34 + .../arrow/converter/UInt1Converter.java | 33 + .../arrow/converter/UInt2Converter.java | 33 + .../arrow/converter/UInt4Converter.java | 33 + .../arrow/converter/UInt8Converter.java | 33 + .../arrow/converter/UnionConverter.java | 33 + .../arrow/converter/VarBinaryConverter.java | 33 + .../arrow/converter/VarCharConverter.java | 33 + .../reader/Arrow2SeatunnelRowReader.java | 264 +++++++ ...el.common.source.arrow.converter.Converter | 60 ++ .../arrow/Arrow2SeatunnelRowReaderTest.java | 463 +++++++++++ .../source/arrow/SeaTunnelDataTypeHolder.java | 84 ++ .../connector-doris/pom.xml | 6 - .../doris/source/reader/DorisValueReader.java | 26 +- .../doris/source/serialization/RowBatch.java | 742 ------------------ .../connector-starrocks/pom.xml | 51 -- .../client/source/StarRocksBeReadClient.java | 8 +- .../source/StarRocksRowBatchReader.java | 357 --------- seatunnel-shade/pom.xml | 3 +- .../pom.xml | 45 +- tools/dependencies/known-dependencies.txt | 10 + 61 files changed, 2615 insertions(+), 1204 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BigIntConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BitConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Converter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateDayConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateMilliConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Decimal256Converter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DecimalConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DenseUnionConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DurationConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ExtensionTypeConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeBinaryConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float4Converter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float8Converter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalDayConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalMonthDayNanoConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalYearConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeListConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarBinaryConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarCharConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ListConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/MapConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/NullConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/SmallIntConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/StructConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMicroConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMilliConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeNanoConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeSecConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroTZConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliTZConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoTZConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecTZConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TinyIntConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt1Converter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt2Converter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt4Converter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt8Converter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UnionConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarBinaryConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarCharConverter.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/Arrow2SeatunnelRowReader.java create mode 100644 seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter create mode 100644 seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/Arrow2SeatunnelRowReaderTest.java create mode 100644 seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/SeaTunnelDataTypeHolder.java delete mode 100644 seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/serialization/RowBatch.java delete mode 100644 seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksRowBatchReader.java rename seatunnel-shade/{seatunnel-arrow-5.0 => seatunnel-arrow}/pom.xml (78%) diff --git a/pom.xml b/pom.xml index e050018b2b8..7c433655921 100644 --- a/pom.xml +++ b/pom.xml @@ -159,6 +159,7 @@ true 3.1.4 + 15.0.1 @@ -489,6 +490,17 @@ provided + + org.apache.arrow + arrow-vector + ${arrow.version} + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + + diff --git a/seatunnel-common/pom.xml b/seatunnel-common/pom.xml index 218ec7dd9d5..a80ca7fbcd0 100644 --- a/seatunnel-common/pom.xml +++ b/seatunnel-common/pom.xml @@ -63,6 +63,13 @@ optional + + org.apache.seatunnel + seatunnel-arrow + ${project.version} + optional + + commons-codec commons-codec diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BigIntConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BigIntConverter.java new file mode 100644 index 00000000000..0b64895e44c --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BigIntConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.BigIntVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class BigIntConverter implements Converter { + @Override + public Object convert(int rowIndex, BigIntVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.BIGINT == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BitConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BitConverter.java new file mode 100644 index 00000000000..b099816642a --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BitConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.BitVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class BitConverter implements Converter { + @Override + public Object convert(int rowIndex, BitVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.BIT == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Converter.java new file mode 100644 index 00000000000..0fdcc08349f --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Converter.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.FieldVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +import java.util.List; +import java.util.function.Function; + +public interface Converter { + + Object convert(int rowIndex, T fieldVector); + + default Object convert(int rowIndex, T fieldVector, List genericsConvert) { + throw new UnsupportedOperationException("Unsupported generics convert"); + } + + boolean support(Types.MinorType type); +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateDayConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateDayConverter.java new file mode 100644 index 00000000000..d62c0d239ee --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateDayConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.DateDayVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class DateDayConverter implements Converter { + @Override + public Object convert(int rowIndex, DateDayVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.DATEDAY == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateMilliConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateMilliConverter.java new file mode 100644 index 00000000000..429c0fae3c3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateMilliConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.DateMilliVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class DateMilliConverter implements Converter { + @Override + public Object convert(int rowIndex, DateMilliVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.DATEMILLI == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Decimal256Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Decimal256Converter.java new file mode 100644 index 00000000000..362a4b70446 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Decimal256Converter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.Decimal256Vector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class Decimal256Converter implements Converter { + @Override + public Object convert(int rowIndex, Decimal256Vector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.DECIMAL256 == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DecimalConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DecimalConverter.java new file mode 100644 index 00000000000..17c5e255c09 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DecimalConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.DecimalVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class DecimalConverter implements Converter { + @Override + public Object convert(int rowIndex, DecimalVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.DECIMAL == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DenseUnionConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DenseUnionConverter.java new file mode 100644 index 00000000000..c60521248d3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DenseUnionConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.DenseUnionVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class DenseUnionConverter implements Converter { + @Override + public Object convert(int rowIndex, DenseUnionVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.DENSEUNION == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DurationConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DurationConverter.java new file mode 100644 index 00000000000..090cec6fce5 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DurationConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.DurationVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class DurationConverter implements Converter { + @Override + public Object convert(int rowIndex, DurationVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.DURATION == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ExtensionTypeConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ExtensionTypeConverter.java new file mode 100644 index 00000000000..7571b2186c5 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ExtensionTypeConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class ExtensionTypeConverter implements Converter { + @Override + public Object convert(int rowIndex, ExtensionTypeVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.EXTENSIONTYPE == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeBinaryConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeBinaryConverter.java new file mode 100644 index 00000000000..e91e8a6d108 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeBinaryConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.FixedSizeBinaryVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class FixedSizeBinaryConverter implements Converter { + @Override + public Object convert(int rowIndex, FixedSizeBinaryVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.FIXEDSIZEBINARY == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java new file mode 100644 index 00000000000..d36ad321f7b --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class FixedSizeListConverter implements Converter { + @Override + public Object convert(int rowIndex, FixedSizeListVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public Object convert( + int rowIndex, FixedSizeListVector fieldVector, List genericsConvert) { + if (fieldVector.isNull(rowIndex)) { + return null; + } + List listData = fieldVector.getObject(rowIndex); + return listData.stream() + .map( + item -> { + if (item instanceof LocalDateTime) { + LocalDateTime localDateTime = + ((LocalDateTime) item) + .atZone(ZoneOffset.UTC) + .withZoneSameInstant(ZoneId.systemDefault()) + .toLocalDateTime(); + return genericsConvert.get(0).apply(localDateTime); + } else { + return genericsConvert.get(0).apply(item); + } + }) + .collect(Collectors.toList()); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.FIXED_SIZE_LIST == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float4Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float4Converter.java new file mode 100644 index 00000000000..e45873ed7c3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float4Converter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.Float4Vector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class Float4Converter implements Converter { + @Override + public Object convert(int rowIndex, Float4Vector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.FLOAT4 == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float8Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float8Converter.java new file mode 100644 index 00000000000..d5322e099a0 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float8Converter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.Float8Vector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class Float8Converter implements Converter { + @Override + public Object convert(int rowIndex, Float8Vector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.FLOAT8 == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntConverter.java new file mode 100644 index 00000000000..77135a8f4ef --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.IntVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class IntConverter implements Converter { + @Override + public Object convert(int rowIndex, IntVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.INT == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalDayConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalDayConverter.java new file mode 100644 index 00000000000..815ae6a4b66 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalDayConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.IntervalDayVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class IntervalDayConverter implements Converter { + @Override + public Object convert(int rowIndex, IntervalDayVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.INTERVALDAY == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalMonthDayNanoConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalMonthDayNanoConverter.java new file mode 100644 index 00000000000..9afe312c9aa --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalMonthDayNanoConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.IntervalMonthDayNanoVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class IntervalMonthDayNanoConverter implements Converter { + @Override + public Object convert(int rowIndex, IntervalMonthDayNanoVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.INTERVALMONTHDAYNANO == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalYearConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalYearConverter.java new file mode 100644 index 00000000000..6f80f8e79cd --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalYearConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.IntervalYearVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class IntervalYearConverter implements Converter { + @Override + public Object convert(int rowIndex, IntervalYearVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.INTERVALYEAR == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeListConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeListConverter.java new file mode 100644 index 00000000000..fe3763a09e0 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeListConverter.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.LargeListVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class LargeListConverter implements Converter { + @Override + public Object convert(int rowIndex, LargeListVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public Object convert( + int rowIndex, LargeListVector fieldVector, List genericsConvert) { + if (fieldVector.isNull(rowIndex)) { + return null; + } + List listData = fieldVector.getObject(rowIndex); + return listData.stream() + .map( + item -> { + if (item instanceof LocalDateTime) { + LocalDateTime localDateTime = + ((LocalDateTime) item) + .atZone(ZoneOffset.UTC) + .withZoneSameInstant(ZoneId.systemDefault()) + .toLocalDateTime(); + return genericsConvert.get(0).apply(localDateTime); + } else { + return genericsConvert.get(0).apply(item); + } + }) + .collect(Collectors.toList()); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.LARGELIST == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarBinaryConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarBinaryConverter.java new file mode 100644 index 00000000000..fa9a42132a1 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarBinaryConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.LargeVarBinaryVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class LargeVarBinaryConverter implements Converter { + @Override + public Object convert(int rowIndex, LargeVarBinaryVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.LARGEVARBINARY == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarCharConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarCharConverter.java new file mode 100644 index 00000000000..6b0d90a638f --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarCharConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.LargeVarCharVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class LargeVarCharConverter implements Converter { + @Override + public Object convert(int rowIndex, LargeVarCharVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex).toString(); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.LARGEVARCHAR == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ListConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ListConverter.java new file mode 100644 index 00000000000..48bf0b722b4 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ListConverter.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.ListVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class ListConverter implements Converter { + @Override + public Object convert(int rowIndex, ListVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public Object convert(int rowIndex, ListVector fieldVector, List genericsConvert) { + if (fieldVector.isNull(rowIndex)) { + return null; + } + List listData = fieldVector.getObject(rowIndex); + return listData.stream() + .map( + item -> { + if (item instanceof LocalDateTime) { + LocalDateTime localDateTime = + ((LocalDateTime) item) + .atZone(ZoneOffset.UTC) + .withZoneSameInstant(ZoneId.systemDefault()) + .toLocalDateTime(); + return genericsConvert.get(0).apply(localDateTime); + } else { + return genericsConvert.get(0).apply(item); + } + }) + .collect(Collectors.toList()); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.LIST == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/MapConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/MapConverter.java new file mode 100644 index 00000000000..9c66d9ac936 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/MapConverter.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.MapVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.impl.UnionMapReader; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class MapConverter implements Converter { + @Override + public Object convert(int rowIndex, MapVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public Object convert(int rowIndex, MapVector fieldVector, List genericsConvert) { + UnionMapReader reader = fieldVector.getReader(); + reader.setPosition(rowIndex); + Map mapValue = new HashMap<>(); + while (reader.next()) { + Object key = genericsConvert.get(0).apply(processTimeZone(reader.key().readObject())); + Object value = + genericsConvert.get(1).apply(processTimeZone(reader.value().readObject())); + mapValue.put(key, value); + } + return mapValue; + } + + private Object processTimeZone(Object value) { + if (value instanceof LocalDateTime) { + return ((LocalDateTime) value) + .atZone(ZoneOffset.UTC) + .withZoneSameInstant(ZoneId.systemDefault()) + .toLocalDateTime(); + } else { + return value; + } + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.MAP == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/NullConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/NullConverter.java new file mode 100644 index 00000000000..50b235ea8ec --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/NullConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.NullVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class NullConverter implements Converter { + @Override + public Object convert(int rowIndex, NullVector fieldVector) { + return null; + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.NULL == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/SmallIntConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/SmallIntConverter.java new file mode 100644 index 00000000000..d634892c66d --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/SmallIntConverter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.SmallIntVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class SmallIntConverter implements Converter { + + @Override + public Object convert(int rowIndex, SmallIntVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.SMALLINT == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/StructConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/StructConverter.java new file mode 100644 index 00000000000..d45c58ad2f6 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/StructConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.StructVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class StructConverter implements Converter { + @Override + public Object convert(int rowIndex, StructVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.STRUCT == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMicroConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMicroConverter.java new file mode 100644 index 00000000000..2e4786a04a5 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMicroConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeMicroVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class TimeMicroConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeMicroVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMEMICRO == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMilliConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMilliConverter.java new file mode 100644 index 00000000000..56a2c0d54bc --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMilliConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeMilliVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class TimeMilliConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeMilliVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMEMILLI == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeNanoConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeNanoConverter.java new file mode 100644 index 00000000000..8cabd2eaf02 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeNanoConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeNanoVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class TimeNanoConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeNanoVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMENANO == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeSecConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeSecConverter.java new file mode 100644 index 00000000000..6d4615d7327 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeSecConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeSecVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class TimeSecConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeSecVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMESEC == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroConverter.java new file mode 100644 index 00000000000..c602d407e4a --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroConverter.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; + +public class TimeStampMicroConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeStampMicroVector fieldVector) { + if (fieldVector == null) { + return null; + } + LocalDateTime localDateTime = fieldVector.getObject(rowIndex); + return localDateTime + .atZone(ZoneOffset.UTC) + .withZoneSameInstant(ZoneId.systemDefault()) + .toLocalDateTime(); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMESTAMPMICRO == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroTZConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroTZConverter.java new file mode 100644 index 00000000000..4eb774f1217 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroTZConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMicroTZVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class TimeStampMicroTZConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeStampMicroTZVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMESTAMPMICROTZ == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliConverter.java new file mode 100644 index 00000000000..f6910d40e3f --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliConverter.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMilliVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; + +public class TimeStampMilliConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeStampMilliVector fieldVector) { + if (fieldVector == null) { + return null; + } + LocalDateTime localDateTime = fieldVector.getObject(rowIndex); + return localDateTime + .atZone(ZoneOffset.UTC) + .withZoneSameInstant(ZoneId.systemDefault()) + .toLocalDateTime(); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMESTAMPMILLI == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliTZConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliTZConverter.java new file mode 100644 index 00000000000..11652819afe --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliTZConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMilliTZVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class TimeStampMilliTZConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeStampMilliTZVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMESTAMPMILLITZ == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoConverter.java new file mode 100644 index 00000000000..17afd9fd1f0 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoConverter.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampNanoVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; + +public class TimeStampNanoConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeStampNanoVector fieldVector) { + if (fieldVector == null) { + return null; + } + LocalDateTime localDateTime = fieldVector.getObject(rowIndex); + return localDateTime + .atZone(ZoneOffset.UTC) + .withZoneSameInstant(ZoneId.systemDefault()) + .toLocalDateTime(); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMESTAMPNANO == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoTZConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoTZConverter.java new file mode 100644 index 00000000000..bd68a3ba94a --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoTZConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampNanoTZVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class TimeStampNanoTZConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeStampNanoTZVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMESTAMPNANOTZ == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java new file mode 100644 index 00000000000..461880fda37 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampSecVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class TimeStampSecConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeStampSecVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMESTAMPSEC == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecTZConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecTZConverter.java new file mode 100644 index 00000000000..ee0e23ab18f --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecTZConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampSecTZVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class TimeStampSecTZConverter implements Converter { + @Override + public Object convert(int rowIndex, TimeStampSecTZVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TIMESTAMPSECTZ == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TinyIntConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TinyIntConverter.java new file mode 100644 index 00000000000..717cd3fccb7 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TinyIntConverter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.TinyIntVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class TinyIntConverter implements Converter { + + @Override + public Object convert(int rowIndex, TinyIntVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.TINYINT == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt1Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt1Converter.java new file mode 100644 index 00000000000..a85d13d9918 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt1Converter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.UInt1Vector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class UInt1Converter implements Converter { + @Override + public Object convert(int rowIndex, UInt1Vector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.UINT1 == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt2Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt2Converter.java new file mode 100644 index 00000000000..9717b2da9f3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt2Converter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.UInt2Vector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class UInt2Converter implements Converter { + @Override + public Object convert(int rowIndex, UInt2Vector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.UINT2 == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt4Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt4Converter.java new file mode 100644 index 00000000000..d34cfcc7218 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt4Converter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.UInt4Vector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class UInt4Converter implements Converter { + @Override + public Object convert(int rowIndex, UInt4Vector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.UINT4 == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt8Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt8Converter.java new file mode 100644 index 00000000000..390924f4bd4 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt8Converter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.UInt8Vector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class UInt8Converter implements Converter { + @Override + public Object convert(int rowIndex, UInt8Vector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.UINT8 == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UnionConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UnionConverter.java new file mode 100644 index 00000000000..5b3b0a953f7 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UnionConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.UnionVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class UnionConverter implements Converter { + @Override + public Object convert(int rowIndex, UnionVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.UNION == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarBinaryConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarBinaryConverter.java new file mode 100644 index 00000000000..1f850cce060 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarBinaryConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.VarBinaryVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class VarBinaryConverter implements Converter { + @Override + public Object convert(int rowIndex, VarBinaryVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.VARBINARY == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarCharConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarCharConverter.java new file mode 100644 index 00000000000..5efd54278fa --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarCharConverter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; + +import org.apache.seatunnel.shade.org.apache.arrow.vector.VarCharVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; + +public class VarCharConverter implements Converter { + @Override + public Object convert(int rowIndex, VarCharVector fieldVector) { + return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex).toString(); + } + + @Override + public boolean support(Types.MinorType type) { + return Types.MinorType.VARCHAR == type; + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/Arrow2SeatunnelRowReader.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/Arrow2SeatunnelRowReader.java new file mode 100644 index 00000000000..a3cb35ded98 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/Arrow2SeatunnelRowReader.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader; + +import org.apache.seatunnel.shade.org.apache.arrow.memory.RootAllocator; +import org.apache.seatunnel.shade.org.apache.arrow.vector.FieldVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.seatunnel.shade.org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; +import org.apache.seatunnel.shade.org.apache.arrow.vector.util.Text; + +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.api.table.type.SqlType; +import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter; + +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.function.Function; + +@Slf4j +public class Arrow2SeatunnelRowReader implements AutoCloseable { + + private final SeaTunnelDataType[] seaTunnelDataTypes; + private int offsetInRowBatch = 0; + private int rowCountInOneBatch = 0; + private int readRowCount = 0; + private List fieldVectors; + private VectorSchemaRoot root; + private final ArrowStreamReader arrowStreamReader; + private final RootAllocator rootAllocator; + private final Map fieldIndexMap = new HashMap<>(); + private final List seatunnelRowBatch = new ArrayList<>(); + private final List converters = new ArrayList<>(); + private final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); + private final DateTimeFormatter DATETIME_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + public Arrow2SeatunnelRowReader(byte[] byteArray, SeaTunnelRowType seaTunnelRowType) { + this.seaTunnelDataTypes = seaTunnelRowType.getFieldTypes(); + this.rootAllocator = new RootAllocator(Integer.MAX_VALUE); + this.arrowStreamReader = + new ArrowStreamReader(new ByteArrayInputStream(byteArray), rootAllocator); + for (int i = 0; i < seaTunnelRowType.getFieldNames().length; i++) { + fieldIndexMap.put(seaTunnelRowType.getFieldNames()[i], i); + } + ServiceLoader load = ServiceLoader.load(Converter.class); + load.forEach(converters::add); + } + + public Arrow2SeatunnelRowReader readArrow() { + try { + this.root = arrowStreamReader.getVectorSchemaRoot(); + while (arrowStreamReader.loadNextBatch()) { + this.fieldVectors = root.getFieldVectors(); + if (fieldVectors.isEmpty() || root.getRowCount() == 0) { + log.debug("one batch in arrow has no data."); + continue; + } + log.info("one batch in arrow row count size '{}'", root.getRowCount()); + this.rowCountInOneBatch = root.getRowCount(); + for (int i = 0; i < rowCountInOneBatch; i++) { + seatunnelRowBatch.add(new SeaTunnelRow(this.seaTunnelDataTypes.length)); + } + convertSeatunnelRow(); + this.readRowCount += root.getRowCount(); + } + return this; + } catch (IOException e) { + throw new RuntimeException(e); + } finally { + close(); + } + } + + public boolean hasNext() { + return offsetInRowBatch < readRowCount; + } + + public SeaTunnelRow next() { + if (!hasNext()) { + throw new IllegalStateException("no more rows to read."); + } + return seatunnelRowBatch.get(offsetInRowBatch++); + } + + private void convertSeatunnelRow() { + for (FieldVector fieldVector : fieldVectors) { + String name = fieldVector.getField().getName(); + Integer fieldIndex = fieldIndexMap.get(name); + Types.MinorType minorType = fieldVector.getMinorType(); + for (int i = 0; i < seatunnelRowBatch.size(); i++) { + // arrow field not in the Seatunnel Sechma field, skip it + if (fieldIndex != null) { + SeaTunnelDataType seaTunnelDataType = seaTunnelDataTypes[fieldIndex]; + Object fieldValue = + convertArrowData( + readRowCount + i, minorType, fieldVector, seaTunnelDataType); + fieldValue = + convertSeatunnelRowValue( + seaTunnelDataType.getSqlType(), minorType, fieldValue); + seatunnelRowBatch.get(readRowCount + i).setField(fieldIndex, fieldValue); + } + } + } + } + + public int getReadRowCount() { + return readRowCount; + } + + private Object convertSeatunnelRowValue( + SqlType currentType, Types.MinorType minorType, Object fieldValue) { + switch (currentType) { + case STRING: + if (fieldValue instanceof byte[]) { + return new String((byte[]) fieldValue); + } else if (fieldValue instanceof Text) { + return ((Text) fieldValue).toString(); + } else { + return fieldValue; + } + case DECIMAL: + if (fieldValue instanceof String) { + return new BigDecimal((String) fieldValue); + } else if (fieldValue instanceof Text) { + return new BigDecimal(((Text) fieldValue).toString()); + } else { + return fieldValue; + } + case DATE: + if (fieldValue instanceof Integer) { + return LocalDate.ofEpochDay((Integer) fieldValue); + } else if (fieldValue instanceof Long) { + return LocalDate.ofEpochDay((Long) fieldValue); + } else if (fieldValue instanceof String) { + return LocalDate.parse((String) fieldValue, DATE_FORMATTER); + } else if (fieldValue instanceof Text) { + return LocalDate.parse(((Text) fieldValue).toString(), DATE_FORMATTER); + } else if (fieldValue instanceof LocalDateTime) { + return ((LocalDateTime) fieldValue).toLocalDate(); + } else { + return fieldValue; + } + case TIME: + if (fieldValue instanceof Integer) { + return LocalTime.ofSecondOfDay((Integer) fieldValue); + } else if (fieldValue instanceof Long) { + return Instant.ofEpochMilli((Long) fieldValue) + .atZone(ZoneId.systemDefault()) + .toLocalDateTime() + .toLocalTime(); + } else if (fieldValue instanceof String) { + return LocalTime.parse((String) fieldValue, TIME_FORMATTER); + } else if (fieldValue instanceof Text) { + return LocalTime.parse(((Text) fieldValue).toString(), TIME_FORMATTER); + } else { + return fieldValue; + } + case TIMESTAMP: + if (fieldValue instanceof Long) { + return Instant.ofEpochMilli((Long) fieldValue) + .atZone(ZoneId.systemDefault()) + .toLocalDateTime(); + } else if (fieldValue instanceof String) { + return LocalDateTime.parse((String) fieldValue, DATETIME_FORMATTER); + } else if (fieldValue instanceof Text) { + return LocalDateTime.parse(((Text) fieldValue).toString(), DATETIME_FORMATTER); + } else { + return fieldValue; + } + default: + return fieldValue; + } + } + + private Object convertArrowData( + int rowIndex, + Types.MinorType minorType, + FieldVector fieldVector, + SeaTunnelDataType seaTunnelDataType) { + for (Converter converter : converters) { + if (converter.support(minorType)) { + SqlType sqlType = seaTunnelDataType.getSqlType(); + if (SqlType.MAP == sqlType) { + MapType mapType = (MapType) seaTunnelDataType; + SqlType keyType = mapType.getKeyType().getSqlType(); + SqlType valueType = mapType.getValueType().getSqlType(); + return converter.convert( + rowIndex, + fieldVector, + Arrays.asList(genericsConvert(keyType), genericsConvert(valueType))); + } else if (SqlType.ARRAY == sqlType) { + ArrayType arrayType = (ArrayType) seaTunnelDataType; + SqlType elementType = arrayType.getElementType().getSqlType(); + return converter.convert( + rowIndex, + fieldVector, + Collections.singletonList(genericsConvert(elementType))); + } else { + return converter.convert(rowIndex, fieldVector); + } + } + } + log.error("No converter found for Arrow MinorType: {}", minorType); + throw new IllegalStateException("Unsupported Arrow MinorType: " + minorType); + } + + private Function genericsConvert(SqlType sqlType) { + return value -> convertSeatunnelRowValue(sqlType, null, value); + } + + @Override + public void close() { + try { + if (root != null) { + root.close(); + } + if (rootAllocator != null) { + rootAllocator.close(); + } + if (arrowStreamReader != null) { + arrowStreamReader.close(); + } + } catch (IOException e) { + throw new RuntimeException("failed to close arrow stream reader.", e); + } + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter b/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter new file mode 100644 index 00000000000..c804b406495 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.BigIntConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.BitConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DateDayConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DateMilliConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Decimal256Converter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DecimalConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DenseUnionConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DurationConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.ExtensionTypeConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.FixedSizeBinaryConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.FixedSizeListConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Float4Converter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Float8Converter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.IntConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.IntervalDayConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.IntervalMonthDayNanoConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.IntervalYearConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.LargeListConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.LargeVarBinaryConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.LargeVarCharConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.ListConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.MapConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.NullConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.SmallIntConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.StructConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeMicroConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeMilliConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeNanoConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeSecConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMicroConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMicroTZConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMilliConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMilliTZConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampNanoConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampNanoTZConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampSecConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampSecTZConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TinyIntConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.UInt1Converter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.UInt2Converter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.UInt4Converter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.UInt8Converter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.UnionConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.VarBinaryConverter +org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.VarCharConverter diff --git a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/Arrow2SeatunnelRowReaderTest.java b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/Arrow2SeatunnelRowReaderTest.java new file mode 100644 index 00000000000..9603e1afc10 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/Arrow2SeatunnelRowReaderTest.java @@ -0,0 +1,463 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow; + +import org.apache.seatunnel.shade.com.google.common.base.Stopwatch; +import org.apache.seatunnel.shade.io.netty.util.CharsetUtil; +import org.apache.seatunnel.shade.org.apache.arrow.memory.ArrowBuf; +import org.apache.seatunnel.shade.org.apache.arrow.memory.BufferAllocator; +import org.apache.seatunnel.shade.org.apache.arrow.memory.RootAllocator; +import org.apache.seatunnel.shade.org.apache.arrow.vector.BigIntVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.BitVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.DateDayVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.DateMilliVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.DecimalVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.FieldVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.Float4Vector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.Float8Vector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.IntVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.LargeVarCharVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.SmallIntVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeMicroVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMicroVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMilliTZVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.TinyIntVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.VarBinaryVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.VarCharVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.ListVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.impl.UnionListWriter; +import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.impl.UnionMapWriter; +import org.apache.seatunnel.shade.org.apache.arrow.vector.holders.TimeMilliHolder; +import org.apache.seatunnel.shade.org.apache.arrow.vector.holders.VarCharHolder; +import org.apache.seatunnel.shade.org.apache.arrow.vector.ipc.ArrowStreamWriter; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.TimeUnit; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.pojo.Field; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.pojo.Schema; + +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader.Arrow2SeatunnelRowReader; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayOutputStream; +import java.math.BigDecimal; +import java.nio.channels.Channels; +import java.nio.charset.StandardCharsets; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +public class Arrow2SeatunnelRowReaderTest { + + private static VectorSchemaRoot root; + private static RootAllocator rootAllocator; + private static final List seaTunnelDataTypeHolder = new ArrayList<>(); + + /** + * LocalDateTime.now() is timestamped with a precision of nanoseconds on linux and milliseconds + * on windows The test case uses TimeStampMicroVector to test the timestamp, thus truncating the + * timestamp accuracy to ChronoUnit.MILLIS + */ + private static final LocalDateTime localDateTime = + LocalDateTime.now().truncatedTo(ChronoUnit.MILLIS); + + private static final List stringData = new ArrayList<>(); + private static final List byteData = new ArrayList<>(); + private static final List shortData = new ArrayList<>(); + private static final List intData = new ArrayList<>(); + private static final List longData = new ArrayList<>(); + private static final float floatData = 1.23f; + private static final double doubleData = 1.23456789d; + private static final BigDecimal decimalData = new BigDecimal("1234567.89"); + private static final List> arrayData1 = new ArrayList<>(); + private static final List> arrayData2 = new ArrayList<>(); + private static final List> mapData = new ArrayList<>(); + + @BeforeAll + public static void beforeAll() throws Exception { + rootAllocator = new RootAllocator(Long.MAX_VALUE); + root = buildVectorSchemaRoot(rootAllocator, 10, true); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("boolean", 1)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("byte", 1)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("short", 1)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("int", 1)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("long", 1)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("float", 1)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("double", 1)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("string1", 1)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("decimal", 1)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("timestamp1", 1)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("string2", 0)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("string3", 0)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("timestamp2", 0)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("time", 0)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("date1", 0)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("date2", 0)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("array1", 0)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("array2", 0)); + seaTunnelDataTypeHolder.add(new SeaTunnelDataTypeHolder("map", 0)); + } + + private static VectorSchemaRoot buildVectorSchemaRoot( + RootAllocator rootAllocator, int count, boolean allType) { + List vectors = new ArrayList<>(); + ZoneId zoneId = ZoneId.systemDefault(); + vectors.add(new BitVector("boolean", rootAllocator)); + vectors.add(new TinyIntVector("byte", rootAllocator)); + vectors.add(new SmallIntVector("short", rootAllocator)); + vectors.add(new IntVector("int", rootAllocator)); + vectors.add(new BigIntVector("long", rootAllocator)); + vectors.add(new Float4Vector("float", rootAllocator)); + vectors.add(new Float8Vector("double", rootAllocator)); + // varchar + vectors.add(new VarCharVector("string1", rootAllocator)); + vectors.add( + new DecimalVector( + Field.nullable("decimal", new ArrowType.Decimal(10, 2, 128)), + rootAllocator)); + // timestamp without timezone + vectors.add(new TimeStampMicroVector("timestamp1", rootAllocator)); + if (allType) { + // byte[] + vectors.add(new VarBinaryVector("string2", rootAllocator)); + // text + vectors.add(new LargeVarCharVector("string3", rootAllocator)); + // timestamp with timezone + vectors.add( + new TimeStampMilliTZVector( + Field.nullable( + "timestamp2", + new ArrowType.Timestamp( + TimeUnit.MILLISECOND, ZoneId.systemDefault().getId())), + rootAllocator)); + vectors.add(new TimeMicroVector("time", rootAllocator)); + vectors.add(new DateMilliVector("date1", rootAllocator)); + vectors.add(new DateDayVector("date2", rootAllocator)); + // array int + vectors.add(ListVector.empty("array1", rootAllocator)); + // array int + vectors.add(ListVector.empty("array2", rootAllocator)); + // map + } + // allocate storage + vectors.forEach(FieldVector::allocateNew); + // setVectorVaule + long epochMilli = + localDateTime + .truncatedTo(ChronoUnit.MILLIS) + .atZone(zoneId) + .toInstant() + .toEpochMilli(); + byte byteStart = 'a'; + + // setVectorValue + vectors.forEach( + vector -> { + for (int i = 0; i < count; i++) { + String stringValue = "test" + i; + if (vector instanceof BitVector) { + ((BitVector) vector).setSafe(i, i % 2 == 0 ? 0 : 1); + } else if (vector instanceof TinyIntVector) { + int i1 = byteStart + i; + byteData.add((byte) i1); + ((TinyIntVector) vector).setSafe(i, i1); + } else if (vector instanceof SmallIntVector) { + shortData.add((short) i); + ((SmallIntVector) vector).setSafe(i, i); + } else if (vector instanceof IntVector) { + intData.add(i); + ((IntVector) vector).setSafe(i, i); + } else if (vector instanceof BigIntVector) { + longData.add((long) i); + ((BigIntVector) vector).setSafe(i, i); + } else if (vector instanceof Float4Vector) { + ((Float4Vector) vector).setSafe(i, floatData); + } else if (vector instanceof Float8Vector) { + ((Float8Vector) vector).setSafe(i, doubleData); + } else if (vector instanceof DecimalVector) { + ((DecimalVector) vector).setSafe(i, decimalData); + } else if (vector instanceof VarCharVector) { + stringData.add(stringValue); + ((VarCharVector) vector) + .setSafe(i, (stringValue).getBytes(StandardCharsets.UTF_8)); + } else if (vector instanceof TimeStampMicroVector) { + ((TimeStampMicroVector) vector).setSafe(i, epochMilli * 1000); + } else if (vector instanceof VarBinaryVector) { + ((VarBinaryVector) vector) + .setSafe(i, (stringValue).getBytes(StandardCharsets.UTF_8)); + } else if (vector instanceof LargeVarCharVector) { + ((LargeVarCharVector) vector) + .setSafe(i, (stringValue).getBytes(StandardCharsets.UTF_8)); + } else if (vector instanceof TimeStampMilliTZVector) { + ((TimeStampMilliTZVector) vector).setSafe(i, epochMilli); + } else if (vector instanceof TimeMicroVector) { + ((TimeMicroVector) vector).setSafe(i, epochMilli); + } else if (vector instanceof DateMilliVector) { + ((DateMilliVector) vector).setSafe(i, epochMilli); + } else if (vector instanceof DateDayVector) { + ((DateDayVector) vector) + .setSafe(i, (int) localDateTime.toLocalDate().toEpochDay()); + } + } + }); + + // setListVectorValue + vectors.stream() + .filter(vector -> vector instanceof ListVector) + .forEach( + vector -> { + ListVector listVector = (ListVector) vector; + String name = listVector.getField().getName(); + UnionListWriter writer = listVector.getWriter(); + for (int i = 0; i < count; i++) { + writer.startList(); + writer.setPosition(i); + if ("array1".equals(name)) { + List intList = new ArrayList<>(); + for (int j = 0; j < 5; j++) { + int i1 = j + i; + writer.writeInt(i1); + intList.add(i1); + } + writer.setValueCount(5); + writer.endList(); + arrayData1.add(intList); + } + if ("array2".equals(name)) { + List dateTimeList = new ArrayList<>(); + for (int j = 0; j < 5; j++) { + writer.writeTimeStampMilliTZ(epochMilli); + dateTimeList.add(localDateTime); + } + writer.setValueCount(5); + writer.endList(); + arrayData2.add(dateTimeList); + } + } + }); + // setMapVectorValue + + // setValueCount + vectors.forEach(vector -> vector.setValueCount(count)); + List fields = + vectors.stream().map(FieldVector::getField).collect(Collectors.toList()); + Schema schema = new Schema(fields); + return new VectorSchemaRoot(schema, vectors, count); + } + + private static void writeKeyAndValue( + UnionMapWriter writer, Object value, int rowIndex, BufferAllocator allocator) { + writer.setPosition(rowIndex); + if (value instanceof String) { + byte[] bytes = ((String) value).getBytes(CharsetUtil.UTF_8); + ArrowBuf buffer = allocator.buffer(bytes.length); + buffer.writeBytes(bytes); + VarCharHolder holder = new VarCharHolder(); + holder.start = 0; + holder.buffer = buffer; + holder.end = bytes.length; + writer.write(holder); + } else if (value instanceof LocalDateTime) { + LocalDateTime dateTime = (LocalDateTime) value; + TimeMilliHolder holder = new TimeMilliHolder(); + holder.value = (int) dateTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + writer.write(holder); + } + } + + @Test + public void testSeatunnelRow() throws Exception { + try (ByteArrayOutputStream out = new ByteArrayOutputStream(); + ArrowStreamWriter writer = + new ArrowStreamWriter( + root, /*DictionaryProvider=*/ null, Channels.newChannel(out))) { + writer.writeBatch(); + out.flush(); + List rows = new ArrayList<>(); + try (Arrow2SeatunnelRowReader reader = + new Arrow2SeatunnelRowReader(out.toByteArray(), getSeatunnelRowType(true)) + .readArrow()) { + while (reader.hasNext()) { + rows.add(reader.next()); + } + Assertions.assertEquals(10, rows.size()); + } + // check boolean + List actualBooleanData = + rows.stream().map(s -> s.getField(0)).distinct().collect(Collectors.toList()); + Assertions.assertEquals(Arrays.asList(Boolean.FALSE, Boolean.TRUE), actualBooleanData); + // check byte + List actualByteData = + rows.stream().map(s -> s.getField(1)).distinct().collect(Collectors.toList()); + Assertions.assertEquals(byteData, actualByteData); + // check short + List actualShortData = + rows.stream().map(s -> s.getField(2)).distinct().collect(Collectors.toList()); + Assertions.assertEquals(shortData, actualShortData); + // check int + List actualIntData = + rows.stream().map(s -> s.getField(3)).distinct().collect(Collectors.toList()); + Assertions.assertEquals(intData, actualIntData); + // check long + List actualLongData = + rows.stream().map(s -> s.getField(4)).distinct().collect(Collectors.toList()); + Assertions.assertEquals(longData, actualLongData); + // check float + List actualFloatData = + rows.stream().map(s -> s.getField(5)).distinct().collect(Collectors.toList()); + Assertions.assertEquals(Collections.singletonList(floatData), actualFloatData); + // check double + List actualDoubleData = + rows.stream().map(s -> s.getField(6)).distinct().collect(Collectors.toList()); + Assertions.assertEquals(Collections.singletonList(doubleData), actualDoubleData); + // check string1 + List actualStringData = + rows.stream().map(s -> s.getField(7)).collect(Collectors.toList()); + Assertions.assertEquals(stringData, actualStringData); + // check decimal + List actualDecimalData = + rows.stream().map(s -> s.getField(8)).distinct().collect(Collectors.toList()); + Assertions.assertEquals(Collections.singletonList(decimalData), actualDecimalData); + // check timestamp without tz + List actualTimestamp1Data = + rows.stream().map(s -> s.getField(9)).distinct().collect(Collectors.toList()); + Assertions.assertEquals(Collections.singletonList(localDateTime), actualTimestamp1Data); + // check string2 + List actualString2Data = + rows.stream().map(s -> s.getField(10)).collect(Collectors.toList()); + Assertions.assertEquals(stringData, actualString2Data); + // check string3 + List actualString3Data = + rows.stream().map(s -> s.getField(11)).collect(Collectors.toList()); + Assertions.assertEquals(stringData, actualString3Data); + + // check timestamp with tz + List actualTimestamp2Data = + rows.stream().map(s -> s.getField(12)).distinct().collect(Collectors.toList()); + Assertions.assertEquals(Collections.singletonList(localDateTime), actualTimestamp2Data); + + // check time + List actualTimeDate = + rows.stream().map(s -> s.getField(13)).distinct().collect(Collectors.toList()); + Assertions.assertEquals( + Collections.singletonList(localDateTime.toLocalTime()), actualTimeDate); + // check date1 + List actualDate1Data = + rows.stream().map(s -> s.getField(14)).distinct().collect(Collectors.toList()); + Assertions.assertEquals( + Collections.singletonList(localDateTime.toLocalDate()), actualDate1Data); + // check date2 + List actualDate2Data = + rows.stream().map(s -> s.getField(15)).distinct().collect(Collectors.toList()); + Assertions.assertEquals( + Collections.singletonList(localDateTime.toLocalDate()), actualDate2Data); + // check array int + List actualArrayIntData = + rows.stream().map(s -> s.getField(16)).collect(Collectors.toList()); + Assertions.assertIterableEquals(arrayData1, actualArrayIntData); + // check array timestamp + List actualArrayTimestampData = + rows.stream().map(s -> s.getField(17)).collect(Collectors.toList()); + Assertions.assertIterableEquals(arrayData2, actualArrayTimestampData); + // todo check map + // The java api has problems building MapVectors,and there are no examples on the + // official website + // @see https://github.com/apache/arrow/issues/44664 + } + } + + @Test + public void testConvertArrowSpeed() throws Exception { + Stopwatch stopwatch = Stopwatch.createStarted(); + int count = 1000000; + try (RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE); + VectorSchemaRoot vectorSchemaRoot = + buildVectorSchemaRoot(rootAllocator, count, false); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ArrowStreamWriter writer = + new ArrowStreamWriter( + vectorSchemaRoot, + /*DictionaryProvider=*/ null, + Channels.newChannel(out))) { + stopwatch.stop(); + System.out.printf( + "build %s rows vectorSchemaRoot cost %s ms \n", + count, stopwatch.elapsed(java.util.concurrent.TimeUnit.MILLISECONDS)); + writer.writeBatch(); + out.flush(); + List rows = new ArrayList<>(); + stopwatch.reset().start(); + SeaTunnelRowType seatunnelRowType = getSeatunnelRowType(false); + try (Arrow2SeatunnelRowReader reader = + new Arrow2SeatunnelRowReader(out.toByteArray(), seatunnelRowType).readArrow()) { + while (reader.hasNext()) { + rows.add(reader.next()); + } + stopwatch.stop(); + System.out.printf( + "read %s rows cost %s ms ", + rows.size(), stopwatch.elapsed(java.util.concurrent.TimeUnit.MILLISECONDS)); + Assertions.assertEquals(count, rows.size()); + } + } + } + + private SeaTunnelRowType getSeatunnelRowType(boolean allType) { + String[] fieldNames = + seaTunnelDataTypeHolder.stream() + .filter(h -> allType ? h.getFlag() >= 0 : h.getFlag() == 1) + .map(SeaTunnelDataTypeHolder::getFiledName) + .toArray(String[]::new); + SeaTunnelDataType[] seaTunnelDataTypes = + seaTunnelDataTypeHolder.stream() + .filter(h -> allType ? h.getFlag() >= 0 : h.getFlag() == 1) + .map(SeaTunnelDataTypeHolder::getSeatunnelDataType) + .toArray(SeaTunnelDataType[]::new); + return new SeaTunnelRowType(fieldNames, seaTunnelDataTypes); + } + + @AfterAll + public static void afterAll() throws Exception { + try { + if (root != null) { + root.close(); + } + if (rootAllocator != null) { + rootAllocator.close(); + } + } catch (Exception e) { + throw new RuntimeException("failed to close arrow stream reader.", e); + } + } +} diff --git a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/SeaTunnelDataTypeHolder.java b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/SeaTunnelDataTypeHolder.java new file mode 100644 index 00000000000..6f1c5579da8 --- /dev/null +++ b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/SeaTunnelDataTypeHolder.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.common.source.arrow; + +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; + +public class SeaTunnelDataTypeHolder { + private final String filedName; + private final int flag; + + public SeaTunnelDataTypeHolder(String filedName, int flag) { + this.filedName = filedName; + this.flag = flag; + } + + public String getFiledName() { + return filedName; + } + + public int getFlag() { + return flag; + } + + public SeaTunnelDataType getSeatunnelDataType() { + switch (filedName) { + case "boolean": + return BasicType.BOOLEAN_TYPE; + case "byte": + return BasicType.BYTE_TYPE; + case "short": + return BasicType.SHORT_TYPE; + case "int": + return BasicType.INT_TYPE; + case "long": + return BasicType.LONG_TYPE; + case "float": + return BasicType.FLOAT_TYPE; + case "double": + return BasicType.DOUBLE_TYPE; + case "string1": + case "string2": + case "string3": + return BasicType.STRING_TYPE; + case "decimal": + return new DecimalType(10, 2); + case "timestamp1": + case "timestamp2": + return LocalTimeType.LOCAL_DATE_TIME_TYPE; + case "time": + return LocalTimeType.LOCAL_TIME_TYPE; + case "date1": + case "date2": + return LocalTimeType.LOCAL_DATE_TYPE; + case "array1": + return ArrayType.INT_ARRAY_TYPE; + case "array2": + return ArrayType.LOCAL_DATE_TIME_ARRAY_TYPE; + case "map": + return new MapType(BasicType.STRING_TYPE, LocalTimeType.LOCAL_DATE_TIME_TYPE); + default: + return null; + } + } +} diff --git a/seatunnel-connectors-v2/connector-doris/pom.xml b/seatunnel-connectors-v2/connector-doris/pom.xml index 85aafc97ad8..06cd650f4a9 100644 --- a/seatunnel-connectors-v2/connector-doris/pom.xml +++ b/seatunnel-connectors-v2/connector-doris/pom.xml @@ -82,11 +82,5 @@ - - org.apache.seatunnel - seatunnel-arrow-5.0 - ${project.version} - optional - diff --git a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/reader/DorisValueReader.java b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/reader/DorisValueReader.java index 68d2eecfb51..deb9b585252 100644 --- a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/reader/DorisValueReader.java +++ b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/reader/DorisValueReader.java @@ -27,8 +27,8 @@ import org.apache.seatunnel.connectors.doris.rest.models.Schema; import org.apache.seatunnel.connectors.doris.source.DorisSourceTable; import org.apache.seatunnel.connectors.doris.source.serialization.Routing; -import org.apache.seatunnel.connectors.doris.source.serialization.RowBatch; import org.apache.seatunnel.connectors.doris.util.SchemaUtils; +import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader.Arrow2SeatunnelRowReader; import org.apache.doris.sdk.thrift.TScanBatchResult; import org.apache.doris.sdk.thrift.TScanCloseParams; @@ -60,12 +60,12 @@ public class DorisValueReader { protected int offset = 0; protected AtomicBoolean eos = new AtomicBoolean(false); - protected RowBatch rowBatch; + protected Arrow2SeatunnelRowReader rowBatch; // flag indicate if support deserialize Arrow to RowBatch asynchronously protected boolean deserializeArrowToRowBatchAsync; - protected BlockingQueue rowBatchBlockingQueue; + protected BlockingQueue rowBatchBlockingQueue; private TScanOpenParams openParams; protected String contextId; protected Schema schema; @@ -115,12 +115,12 @@ private BackendClient backendClient() { private TScanOpenParams openParams() { TScanOpenParams params = new TScanOpenParams(); - params.cluster = DORIS_DEFAULT_CLUSTER; - params.database = partition.getDatabase(); - params.table = partition.getTable(); + params.setCluster(DORIS_DEFAULT_CLUSTER); + params.setDatabase(partition.getDatabase()); + params.setTable(partition.getTable()); - params.tablet_ids = Arrays.asList(partition.getTabletIds().toArray(new Long[] {})); - params.opaqued_query_plan = partition.getQueryPlan(); + params.setTabletIds(Arrays.asList(partition.getTabletIds().toArray(new Long[] {}))); + params.setOpaquedQueryPlan(partition.getQueryPlan()); // max row number of one read batch Integer batchSize = dorisSourceTable.getBatchSize(); Integer queryDorisTimeout = config.getRequestQueryTimeoutS(); @@ -158,8 +158,10 @@ public void run() { TScanBatchResult nextResult = client.getNext(nextBatchParams); eos.set(nextResult.isEos()); if (!eos.get()) { - RowBatch rowBatch = - new RowBatch(nextResult, seaTunnelRowType) + Arrow2SeatunnelRowReader rowBatch = + new Arrow2SeatunnelRowReader( + nextResult.getRows(), + seaTunnelRowType) .readArrow(); offset += rowBatch.getReadRowCount(); rowBatch.close(); @@ -233,7 +235,9 @@ public boolean hasNext() { TScanBatchResult nextResult = client.getNext(nextBatchParams); eos.set(nextResult.isEos()); if (!eos.get()) { - rowBatch = new RowBatch(nextResult, seaTunnelRowType).readArrow(); + rowBatch = + new Arrow2SeatunnelRowReader(nextResult.getRows(), seaTunnelRowType) + .readArrow(); } } hasNext = !eos.get(); diff --git a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/serialization/RowBatch.java b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/serialization/RowBatch.java deleted file mode 100644 index a1f1c678aa4..00000000000 --- a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/serialization/RowBatch.java +++ /dev/null @@ -1,742 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.doris.source.serialization; - -import org.apache.seatunnel.shade.com.google.common.base.Preconditions; -import org.apache.seatunnel.shade.org.apache.arrow.memory.RootAllocator; -import org.apache.seatunnel.shade.org.apache.arrow.vector.BigIntVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.BitVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.DateDayVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.DecimalVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.FieldVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.FixedSizeBinaryVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.Float4Vector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.Float8Vector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.IntVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.SmallIntVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMicroVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.TinyIntVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.VarCharVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.ListVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.MapVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.StructVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.impl.UnionMapReader; -import org.apache.seatunnel.shade.org.apache.arrow.vector.ipc.ArrowStreamReader; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types.MinorType; -import org.apache.seatunnel.shade.org.apache.arrow.vector.util.Text; - -import org.apache.seatunnel.api.table.type.ArrayType; -import org.apache.seatunnel.api.table.type.DecimalArrayType; -import org.apache.seatunnel.api.table.type.MapType; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.api.table.type.SqlType; -import org.apache.seatunnel.common.utils.DateTimeUtils; -import org.apache.seatunnel.common.utils.DateUtils; -import org.apache.seatunnel.connectors.doris.exception.DorisConnectorErrorCode; -import org.apache.seatunnel.connectors.doris.exception.DorisConnectorException; - -import org.apache.doris.sdk.thrift.TScanBatchResult; - -import lombok.extern.slf4j.Slf4j; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.function.IntFunction; - -@Slf4j -public class RowBatch { - SeaTunnelDataType[] fieldTypes; - private final ArrowStreamReader arrowStreamReader; - private final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss"; - private final String DATETIMEV2_PATTERN = "yyyy-MM-dd HH:mm:ss.SSSSSS"; - private final DateTimeFormatter dateTimeV2Formatter = - DateTimeFormatter.ofPattern(DATETIMEV2_PATTERN); - private final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - // offset for iterate the rowBatch - private int offsetInRowBatch = 0; - private int rowCountInOneBatch = 0; - private int readRowCount = 0; - private List seatunnelRowBatch = new ArrayList<>(); - private VectorSchemaRoot root; - private List fieldVectors; - private RootAllocator rootAllocator; - - public RowBatch(TScanBatchResult nextResult, SeaTunnelRowType seaTunnelRowType) { - this.rootAllocator = new RootAllocator(Integer.MAX_VALUE); - this.arrowStreamReader = - new ArrowStreamReader( - new ByteArrayInputStream(nextResult.getRows()), rootAllocator); - this.offsetInRowBatch = 0; - this.fieldTypes = seaTunnelRowType.getFieldTypes(); - } - - public RowBatch readArrow() { - try { - this.root = arrowStreamReader.getVectorSchemaRoot(); - while (arrowStreamReader.loadNextBatch()) { - fieldVectors = root.getFieldVectors(); - // Adapt unique model hidden columns - for (int i = 0; i < fieldVectors.size(); i++) { - String fieldName = fieldVectors.get(i).getField().getName(); - if (fieldName.equals("__DORIS_DELETE_SIGN__")) { - fieldVectors.remove(fieldVectors.get(i)); - } - } - if (fieldVectors.size() != fieldTypes.length) { - log.error( - "Schema size '{}' is not equal to arrow field size '{}'.", - fieldVectors.size(), - fieldTypes.length); - throw new DorisConnectorException( - DorisConnectorErrorCode.ARROW_READ_FAILED, - "Load Doris data failed, schema size of fetch data is wrong."); - } - if (fieldVectors.size() == 0 || root.getRowCount() == 0) { - log.debug("One batch in arrow has no data."); - continue; - } - rowCountInOneBatch = root.getRowCount(); - // init the rowBatch - for (int i = 0; i < rowCountInOneBatch; ++i) { - seatunnelRowBatch.add((new SeaTunnelRow(fieldVectors.size()))); - } - convertArrowToRowBatch(); - readRowCount += root.getRowCount(); - } - return this; - } catch (Throwable e) { - log.error("Read Doris Data failed because: ", e); - throw new DorisConnectorException( - DorisConnectorErrorCode.ARROW_READ_FAILED, e.getMessage()); - } finally { - close(); - } - } - - public boolean hasNext() { - return offsetInRowBatch < readRowCount; - } - - private void addValueToRow(int rowIndex, int colIndex, Object obj) { - if (rowIndex > rowCountInOneBatch) { - String errMsg = - "Get row offset: " + rowIndex + " larger than row size: " + rowCountInOneBatch; - log.error(errMsg); - throw new NoSuchElementException(errMsg); - } - seatunnelRowBatch.get(readRowCount + rowIndex).setField(colIndex, obj); - } - - public void convertArrowToRowBatch() throws DorisConnectorException { - try { - for (int col = 0; col < fieldVectors.size(); col++) { - SeaTunnelDataType dataType = fieldTypes[col]; - final String currentType = dataType.getSqlType().name(); - - FieldVector fieldVector = fieldVectors.get(col); - Types.MinorType minorType = fieldVector.getMinorType(); - convertArrowValue(col, currentType, dataType, minorType, fieldVector); - } - } catch (Throwable e) { - close(); - throw e; - } - } - - private void convertArrowValue( - int col, - String currentType, - SeaTunnelDataType dataType, - MinorType minorType, - FieldVector fieldVector) { - switch (currentType) { - case "BOOLEAN": - BitVector bitVector = (BitVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.BIT), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> - bitVector.isNull(rowIndex) ? null : bitVector.get(rowIndex) != 0); - break; - case "TINYINT": - TinyIntVector tinyIntVector = (TinyIntVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.TINYINT), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> - tinyIntVector.isNull(rowIndex) - ? null - : tinyIntVector.get(rowIndex)); - break; - case "SMALLINT": - if (fieldVector instanceof BitVector) { - BitVector bv = (BitVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.BIT), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, rowIndex -> bv.isNull(rowIndex) ? null : (short) bv.get(rowIndex)); - - } else if (fieldVector instanceof TinyIntVector) { - TinyIntVector tv = (TinyIntVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(MinorType.TINYINT), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, rowIndex -> tv.isNull(rowIndex) ? null : (short) tv.get(rowIndex)); - - } else { - SmallIntVector smallIntVector = (SmallIntVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.SMALLINT), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> - smallIntVector.isNull(rowIndex) - ? null - : smallIntVector.get(rowIndex)); - } - break; - case "INT": - IntVector intVector = (IntVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.INT), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> intVector.isNull(rowIndex) ? null : intVector.get(rowIndex)); - break; - case "BIGINT": - BigIntVector bigIntVector = (BigIntVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.BIGINT), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> - bigIntVector.isNull(rowIndex) ? null : bigIntVector.get(rowIndex)); - break; - case "FLOAT": - Float4Vector float4Vector = (Float4Vector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.FLOAT4), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> - float4Vector.isNull(rowIndex) ? null : float4Vector.get(rowIndex)); - break; - case "DOUBLE": - Float8Vector float8Vector = (Float8Vector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.FLOAT8), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> - float8Vector.isNull(rowIndex) ? null : float8Vector.get(rowIndex)); - break; - case "DECIMAL": - // LARGEINT - if (fieldVector instanceof FixedSizeBinaryVector) { - FixedSizeBinaryVector largeInitFixedSizeBinaryVector = - (FixedSizeBinaryVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.FIXEDSIZEBINARY), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> { - if (largeInitFixedSizeBinaryVector.isNull(rowIndex)) { - return null; - } - byte[] bytes = largeInitFixedSizeBinaryVector.get(rowIndex); - int left = 0, right = bytes.length - 1; - while (left < right) { - byte temp = bytes[left]; - bytes[left] = bytes[right]; - bytes[right] = temp; - left++; - right--; - } - return new BigDecimal(new BigInteger(bytes), 0); - }); - break; - } else if (fieldVector instanceof VarCharVector) { - VarCharVector varCharVector = (VarCharVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.VARCHAR), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> - varCharVector.isNull(rowIndex) - ? null - : new BigDecimal( - new String(varCharVector.get(rowIndex)))); - break; - } - DecimalVector decimalVector = (DecimalVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.DECIMAL), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> - decimalVector.isNull(rowIndex) - ? null - : decimalVector.getObject(rowIndex).stripTrailingZeros()); - break; - case "DATE": - case "DATEV2": - if (fieldVector instanceof DateDayVector) { - DateDayVector dateVector = (DateDayVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.DATEDAY), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> { - if (dateVector.isNull(rowIndex)) { - return null; - } - return LocalDate.ofEpochDay(dateVector.get(rowIndex)); - }); - break; - } - VarCharVector dateVector = (VarCharVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.VARCHAR), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> { - if (dateVector.isNull(rowIndex)) { - return null; - } - String stringValue = new String(dateVector.get(rowIndex)); - return LocalDate.parse(stringValue, dateFormatter); - }); - break; - case "TIMESTAMP": - if (fieldVector instanceof TimeStampMicroVector) { - TimeStampMicroVector timestampVector = (TimeStampMicroVector) fieldVector; - - addValueToRowForAllRows( - col, - rowIndex -> { - if (timestampVector.isNull(rowIndex)) { - return null; - } - String stringValue = timestampVector.getObject(rowIndex).toString(); - stringValue = completeMilliseconds(stringValue); - - return DateTimeUtils.parse(stringValue); - }); - break; - } - VarCharVector timestampVector = (VarCharVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.VARCHAR), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> { - if (timestampVector.isNull(rowIndex)) { - return null; - } - String stringValue = new String(timestampVector.get(rowIndex)); - stringValue = completeMilliseconds(stringValue); - return LocalDateTime.parse(stringValue, dateTimeV2Formatter); - }); - break; - case "STRING": - if (fieldVector instanceof FixedSizeBinaryVector) { - FixedSizeBinaryVector fixedSizeBinaryVector = - (FixedSizeBinaryVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.FIXEDSIZEBINARY), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> { - if (fixedSizeBinaryVector.isNull(rowIndex)) { - return null; - } - byte[] bytes = fixedSizeBinaryVector.get(rowIndex); - int left = 0, right = bytes.length - 1; - while (left < right) { - byte temp = bytes[left]; - bytes[left] = bytes[right]; - bytes[right] = temp; - left++; - right--; - } - return new BigInteger(bytes).toString(); - }); - break; - } else if (fieldVector instanceof MapVector) { - MapVector mapVector = (MapVector) fieldVector; - UnionMapReader reader = mapVector.getReader(); - Preconditions.checkArgument( - minorType.equals(MinorType.MAP), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> { - if (mapVector.isNull(rowIndex)) { - return null; - } - reader.setPosition(rowIndex); - Map mapValue = new HashMap<>(); - while (reader.next()) { - mapValue.put( - reader.key().readObject().toString(), - reader.value().readObject()); - } - return mapValue.toString(); - }); - } else if (fieldVector instanceof StructVector) { - StructVector structVector = (StructVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(MinorType.STRUCT), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> { - if (structVector.isNull(rowIndex)) { - return null; - } - Map structValue = structVector.getObject(rowIndex); - return structValue.toString(); - }); - } else if (fieldVector instanceof ListVector) { - ListVector listVector = (ListVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.LIST), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> { - if (listVector.isNull(rowIndex)) { - return null; - } - List listVectorObject = listVector.getObject(rowIndex); - return Arrays.toString(listVectorObject.toArray()); - }); - } else { - VarCharVector varCharVector = (VarCharVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.VARCHAR), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> - varCharVector.isNull(rowIndex) - ? null - : new String(varCharVector.get(rowIndex))); - } - break; - case "ARRAY": - ListVector listVector = (ListVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(Types.MinorType.LIST), - typeMismatchMessage(currentType, minorType)); - addValueToRowForArrayColumn(dataType, col, listVector); - break; - case "MAP": - MapVector mapVector = (MapVector) fieldVector; - UnionMapReader reader = mapVector.getReader(); - Preconditions.checkArgument( - minorType.equals(MinorType.MAP), - typeMismatchMessage(currentType, minorType)); - addValueToRowForMapColumn(dataType, col, mapVector, reader); - - break; - case "STRUCT": - StructVector structVector = (StructVector) fieldVector; - Preconditions.checkArgument( - minorType.equals(MinorType.STRUCT), - typeMismatchMessage(currentType, minorType)); - addValueToRowForAllRows( - col, - rowIndex -> { - if (structVector.isNull(rowIndex)) { - return null; - } - Map structValue = structVector.getObject(rowIndex); - return structValue; - }); - break; - default: - String errMsg = "Unsupported type " + fieldTypes[col].getSqlType().name(); - log.error(errMsg); - throw new DorisConnectorException( - DorisConnectorErrorCode.ARROW_READ_FAILED, errMsg); - } - } - - private void addValueToRowForMapColumn( - SeaTunnelDataType dataType, int col, MapVector mapVector, UnionMapReader reader) { - addValueToRowForAllRows( - col, - rowIndex -> { - if (mapVector.isNull(rowIndex)) { - return null; - } - reader.setPosition(rowIndex); - Map mapValue = new HashMap<>(); - MapType mapType = (MapType) dataType; - SqlType keyType = mapType.getKeyType().getSqlType(); - SqlType valueType = mapType.getValueType().getSqlType(); - while (reader.next()) { - mapValue.put( - getDataFromVector(reader.key().readObject(), keyType), - getDataFromVector(reader.value().readObject(), valueType)); - } - return mapValue; - }); - } - - private Object getDataFromVector(Object vectorObject, SqlType sqlType) { - if (vectorObject instanceof Boolean) { - return Boolean.valueOf(vectorObject.toString()); - } - - if (vectorObject instanceof Byte) { - return Byte.valueOf(vectorObject.toString()); - } - - if (vectorObject instanceof Short) { - return Short.valueOf(vectorObject.toString()); - } - - if (vectorObject instanceof Integer) { - if (sqlType.equals(SqlType.DATE)) { - return LocalDate.ofEpochDay((int) vectorObject); - } - return Integer.valueOf(vectorObject.toString()); - } - - if (vectorObject instanceof Long) { - return Long.valueOf(vectorObject.toString()); - } - - if (vectorObject instanceof Float) { - return Float.valueOf(vectorObject.toString()); - } - - if (vectorObject instanceof Double) { - return Double.valueOf(vectorObject.toString()); - } - - if (vectorObject instanceof Text) { - if (sqlType.equals(SqlType.TIMESTAMP)) { - String stringValue = completeMilliseconds(vectorObject.toString()); - return LocalDateTime.parse(stringValue, dateTimeV2Formatter); - } else if (sqlType.equals(SqlType.DATE)) { - return LocalDate.parse(vectorObject.toString(), dateFormatter); - } else if (sqlType.equals(SqlType.DECIMAL)) { - return new BigDecimal(vectorObject.toString()); - } - return vectorObject.toString(); - } - - if (vectorObject instanceof BigDecimal) { - return new BigDecimal(vectorObject.toString()); - } - - if (vectorObject instanceof byte[] && sqlType.equals(SqlType.DECIMAL)) { - byte[] bytes = (byte[]) vectorObject; - int left = 0, right = bytes.length - 1; - while (left < right) { - byte temp = bytes[left]; - bytes[left] = bytes[right]; - bytes[right] = temp; - left++; - right--; - } - return new BigDecimal(new BigInteger(bytes), 0); - } - if (vectorObject instanceof LocalDate) { - return DateUtils.parse(vectorObject.toString()); - } - - if (vectorObject instanceof LocalDateTime) { - return DateTimeUtils.parse(vectorObject.toString()); - } - - return vectorObject.toString(); - } - - private void addValueToRowForArrayColumn( - SeaTunnelDataType dataType, int col, ListVector listVector) { - SqlType eleSqlType = null; - if (dataType instanceof ArrayType) { - ArrayType arrayType = (ArrayType) dataType; - eleSqlType = arrayType.getElementType().getSqlType(); - } - SqlType finalEleSqlType = eleSqlType; - addValueToRowForAllRows( - col, - rowIndex -> { - if (listVector.isNull(rowIndex)) { - return null; - } - List listVectorObject = listVector.getObject(rowIndex); - if (listVectorObject.get(0) instanceof Boolean) { - return listVectorObject.stream() - .map(x -> getDataFromVector(x, finalEleSqlType)) - .toArray(Boolean[]::new); - } - - if (listVectorObject.get(0) instanceof Byte) { - return listVectorObject.stream() - .map(x -> getDataFromVector(x, finalEleSqlType)) - .toArray(Byte[]::new); - } - - if (listVectorObject.get(0) instanceof Short) { - return listVectorObject.stream() - .map(x -> getDataFromVector(x, finalEleSqlType)) - .toArray(Short[]::new); - } - - if (listVectorObject.get(0) instanceof Integer) { - return listVectorObject.stream() - .map(x -> getDataFromVector(x, finalEleSqlType)) - .toArray(Integer[]::new); - } - - if (listVectorObject.get(0) instanceof Long) { - return listVectorObject.stream() - .map(x -> getDataFromVector(x, finalEleSqlType)) - .toArray(Long[]::new); - } - - if (listVectorObject.get(0) instanceof Float) { - return listVectorObject.stream() - .map(x -> getDataFromVector(x, finalEleSqlType)) - .toArray(Float[]::new); - } - - if (listVectorObject.get(0) instanceof Double) { - return listVectorObject.stream() - .map(x -> getDataFromVector(x, finalEleSqlType)) - .toArray(Double[]::new); - } - - if (listVectorObject.get(0) instanceof Text) { - return listVectorObject.stream() - .map(x -> getDataFromVector(x, finalEleSqlType)) - .toArray(String[]::new); - } - - if (listVectorObject.get(0) instanceof BigDecimal) { - return listVectorObject.stream() - .map(x -> getDataFromVector(x, finalEleSqlType)) - .toArray(BigDecimal[]::new); - } - - if (listVectorObject.get(0) instanceof byte[] - && dataType instanceof DecimalArrayType) { - return listVectorObject.stream() - .map(x -> getDataFromVector(x, finalEleSqlType)) - .toArray(BigDecimal[]::new); - } - - return listVectorObject.toArray(); - }); - } - - private void addValueToRowForAllRows(int col, IntFunction function) { - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - Object fieldValue = function.apply(rowIndex); - addValueToRow(rowIndex, col, fieldValue); - } - } - - private String completeMilliseconds(String stringValue) { - if (stringValue.length() == DATETIMEV2_PATTERN.length()) { - return stringValue; - } - StringBuilder sb = new StringBuilder(stringValue); - if (stringValue.length() == DATETIME_PATTERN.length()) { - sb.append("."); - } - while (sb.toString().length() < DATETIMEV2_PATTERN.length()) { - sb.append(0); - } - return sb.toString(); - } - - public SeaTunnelRow next() { - if (!hasNext()) { - String errMsg = - "Get row offset:" + offsetInRowBatch + " larger than row size: " + readRowCount; - log.error(errMsg); - throw new NoSuchElementException(errMsg); - } - return seatunnelRowBatch.get(offsetInRowBatch++); - } - - private String typeMismatchMessage(final String flinkType, final Types.MinorType arrowType) { - final String messageTemplate = "FLINK type is %1$s, but arrow type is %2$s."; - return String.format(messageTemplate, flinkType, arrowType.name()); - } - - public int getReadRowCount() { - return readRowCount; - } - - public void close() { - try { - if (arrowStreamReader != null) { - arrowStreamReader.close(); - } - if (rootAllocator != null) { - rootAllocator.close(); - } - } catch (IOException ioe) { - throw new DorisConnectorException( - DorisConnectorErrorCode.ROW_BATCH_GET_FAILED, - "Failed to close ArrowStreamReader", - ioe); - } - } -} diff --git a/seatunnel-connectors-v2/connector-starrocks/pom.xml b/seatunnel-connectors-v2/connector-starrocks/pom.xml index 08e49bc0f02..6653610de4e 100644 --- a/seatunnel-connectors-v2/connector-starrocks/pom.xml +++ b/seatunnel-connectors-v2/connector-starrocks/pom.xml @@ -55,11 +55,6 @@ ${mysql.version} provided - - org.apache.seatunnel - seatunnel-common - ${project.version} - org.apache.httpcomponents httpclient @@ -75,52 +70,6 @@ starrocks-thrift-sdk ${starrocks.thrift.sdk.version} - - org.apache.arrow - arrow-vector - ${arrow.version} - - - org.apache.arrow - arrow-memory-netty - ${arrow.version} - - - - - org.apache.maven.plugins - maven-shade-plugin - - - - org.apache.arrow - ${seatunnel.shade.package}.${connector.name}.org.apache.arrow - - - io.netty - ${seatunnel.shade.package}.${connector.name}.io.netty - - - com.google.flatbuffers - ${seatunnel.shade.package}.${connector.name}.com.google.flatbuffers - - - com.fasterxml.jackson - ${seatunnel.shade.package}.${connector.name}.com.fasterxml.jackson - - - - - - - shade - - package - - - - - diff --git a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksBeReadClient.java b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksBeReadClient.java index 07a5a03eba7..6a1e90dfa0c 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksBeReadClient.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksBeReadClient.java @@ -19,6 +19,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader.Arrow2SeatunnelRowReader; import org.apache.seatunnel.connectors.seatunnel.starrocks.client.source.model.QueryPartition; import org.apache.seatunnel.connectors.seatunnel.starrocks.config.SourceConfig; import org.apache.seatunnel.connectors.seatunnel.starrocks.exception.StarRocksConnectorErrorCode; @@ -56,7 +57,7 @@ public class StarRocksBeReadClient implements Serializable { private int readerOffset = 0; private final SourceConfig sourceConfig; private SeaTunnelRowType seaTunnelRowType; - private StarRocksRowBatchReader rowBatch; + private Arrow2SeatunnelRowReader rowBatch; protected AtomicBoolean eos = new AtomicBoolean(false); public StarRocksBeReadClient(String beNodeInfo, SourceConfig sourceConfig) { @@ -162,7 +163,10 @@ public boolean hasNext() { } eos.set(result.isEos()); if (!eos.get()) { - rowBatch = new StarRocksRowBatchReader(result, seaTunnelRowType).readArrow(); + + rowBatch = + new Arrow2SeatunnelRowReader(result.getRows(), seaTunnelRowType) + .readArrow(); } } catch (TException e) { throw new StarRocksConnectorException( diff --git a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksRowBatchReader.java b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksRowBatchReader.java deleted file mode 100644 index 2ea7e98c7e5..00000000000 --- a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksRowBatchReader.java +++ /dev/null @@ -1,357 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.seatunnel.connectors.seatunnel.starrocks.client.source; - -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.connectors.seatunnel.starrocks.exception.StarRocksConnectorErrorCode; -import org.apache.seatunnel.connectors.seatunnel.starrocks.exception.StarRocksConnectorException; - -import org.apache.arrow.memory.RootAllocator; -import org.apache.arrow.vector.BigIntVector; -import org.apache.arrow.vector.BitVector; -import org.apache.arrow.vector.DecimalVector; -import org.apache.arrow.vector.FieldVector; -import org.apache.arrow.vector.Float4Vector; -import org.apache.arrow.vector.Float8Vector; -import org.apache.arrow.vector.IntVector; -import org.apache.arrow.vector.SmallIntVector; -import org.apache.arrow.vector.TinyIntVector; -import org.apache.arrow.vector.VarCharVector; -import org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.arrow.vector.ipc.ArrowStreamReader; -import org.apache.arrow.vector.types.Types; - -import com.starrocks.thrift.TScanBatchResult; -import lombok.extern.slf4j.Slf4j; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.math.BigDecimal; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; - -import static org.apache.seatunnel.shade.com.google.common.base.Preconditions.checkArgument; - -@Slf4j -public class StarRocksRowBatchReader { - - private SeaTunnelDataType[] seaTunnelDataTypes; - private int offsetInRowBatch = 0; - private int rowCountInOneBatch = 0; - private int readRowCount = 0; - private List seaTunnelRowBatch = new ArrayList<>(); - private final ArrowStreamReader arrowStreamReader; - private VectorSchemaRoot root; - private List fieldVectors; - private RootAllocator rootAllocator; - private final DateTimeFormatter dateTimeFormatter = - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - private final DateTimeFormatter dateFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - - public StarRocksRowBatchReader(TScanBatchResult nextResult, SeaTunnelRowType seaTunnelRowType) { - this.seaTunnelDataTypes = seaTunnelRowType.getFieldTypes(); - this.rootAllocator = new RootAllocator(Integer.MAX_VALUE); - this.arrowStreamReader = - new ArrowStreamReader( - new ByteArrayInputStream(nextResult.getRows()), rootAllocator); - } - - public StarRocksRowBatchReader readArrow() { - try { - this.root = arrowStreamReader.getVectorSchemaRoot(); - while (arrowStreamReader.loadNextBatch()) { - fieldVectors = root.getFieldVectors(); - if (fieldVectors.size() != seaTunnelDataTypes.length) { - log.error( - "seaTunnel schema size '{}' is not equal to arrow field size '{}'.", - fieldVectors.size(), - seaTunnelDataTypes.length); - throw new StarRocksConnectorException( - StarRocksConnectorErrorCode.READER_ARROW_DATA_FAILED, - "schema size of fetch data is wrong."); - } - if (fieldVectors.size() == 0 || root.getRowCount() == 0) { - log.debug("one batch in arrow has no data."); - continue; - } - log.info("one batch in arrow row count size '{}'", root.getRowCount()); - rowCountInOneBatch = root.getRowCount(); - // init the rowBatch - for (int i = 0; i < rowCountInOneBatch; ++i) { - seaTunnelRowBatch.add(new SeaTunnelRow(fieldVectors.size())); - } - - convertArrowToRowBatch(); - readRowCount += root.getRowCount(); - } - return this; - } catch (Exception e) { - throw new StarRocksConnectorException( - StarRocksConnectorErrorCode.READER_ARROW_DATA_FAILED, e); - } finally { - close(); - } - } - - public boolean hasNext() { - if (offsetInRowBatch < readRowCount) { - return true; - } - return false; - } - - private void addValueToRow(int rowIndex, int colIndex, Object obj) { - if (rowIndex > rowCountInOneBatch) { - throw new StarRocksConnectorException( - StarRocksConnectorErrorCode.READER_ARROW_DATA_FAILED, - String.format( - "Get row offset: %d larger than row size: %d", - rowIndex, rowCountInOneBatch)); - } - seaTunnelRowBatch.get(readRowCount + rowIndex).setField(colIndex, obj); - } - - public void convertArrowToRowBatch() { - try { - for (int col = 0; col < fieldVectors.size(); col++) { - SeaTunnelDataType dataType = seaTunnelDataTypes[col]; - final String currentType = dataType.getSqlType().name(); - - FieldVector curFieldVector = fieldVectors.get(col); - Types.MinorType mt = curFieldVector.getMinorType(); - switch (dataType.getSqlType()) { - case BOOLEAN: - checkArgument( - mt.equals(Types.MinorType.BIT), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - BitVector bitVector = (BitVector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - Object fieldValue = - bitVector.isNull(rowIndex) - ? null - : bitVector.get(rowIndex) != 0; - addValueToRow(rowIndex, col, fieldValue); - } - break; - case TINYINT: - checkArgument( - mt.equals(Types.MinorType.TINYINT), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - TinyIntVector tinyIntVector = (TinyIntVector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - Object fieldValue = - tinyIntVector.isNull(rowIndex) - ? null - : tinyIntVector.get(rowIndex); - addValueToRow(rowIndex, col, fieldValue); - } - break; - case SMALLINT: - checkArgument( - mt.equals(Types.MinorType.SMALLINT), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - SmallIntVector smallIntVector = (SmallIntVector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - Object fieldValue = - smallIntVector.isNull(rowIndex) - ? null - : smallIntVector.get(rowIndex); - addValueToRow(rowIndex, col, fieldValue); - } - break; - case INT: - checkArgument( - mt.equals(Types.MinorType.INT), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - IntVector intVector = (IntVector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - Object fieldValue = - intVector.isNull(rowIndex) ? null : intVector.get(rowIndex); - addValueToRow(rowIndex, col, fieldValue); - } - break; - case BIGINT: - checkArgument( - mt.equals(Types.MinorType.BIGINT), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - BigIntVector bigIntVector = (BigIntVector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - Object fieldValue = - bigIntVector.isNull(rowIndex) - ? null - : bigIntVector.get(rowIndex); - addValueToRow(rowIndex, col, fieldValue); - } - break; - case FLOAT: - checkArgument( - mt.equals(Types.MinorType.FLOAT4), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - Float4Vector float4Vector = (Float4Vector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - Object fieldValue = - float4Vector.isNull(rowIndex) - ? null - : float4Vector.get(rowIndex); - addValueToRow(rowIndex, col, fieldValue); - } - break; - case DOUBLE: - checkArgument( - mt.equals(Types.MinorType.FLOAT8), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - Float8Vector float8Vector = (Float8Vector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - Object fieldValue = - float8Vector.isNull(rowIndex) - ? null - : float8Vector.get(rowIndex); - addValueToRow(rowIndex, col, fieldValue); - } - break; - case DECIMAL: - checkArgument( - mt.equals(Types.MinorType.DECIMAL), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - DecimalVector decimalVector = (DecimalVector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - if (decimalVector.isNull(rowIndex)) { - addValueToRow(rowIndex, col, null); - continue; - } - BigDecimal value = decimalVector.getObject(rowIndex); - addValueToRow(rowIndex, col, value); - } - break; - case DATE: - checkArgument( - mt.equals(Types.MinorType.VARCHAR), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - VarCharVector date = (VarCharVector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - if (date.isNull(rowIndex)) { - addValueToRow(rowIndex, col, null); - continue; - } - String value = new String(date.get(rowIndex)); - LocalDate localDate = LocalDate.parse(value, dateFormatter); - addValueToRow(rowIndex, col, localDate); - } - break; - case TIMESTAMP: - checkArgument( - mt.equals(Types.MinorType.VARCHAR), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - VarCharVector timeStampSecVector = (VarCharVector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - if (timeStampSecVector.isNull(rowIndex)) { - addValueToRow(rowIndex, col, null); - continue; - } - String value = new String(timeStampSecVector.get(rowIndex)); - LocalDateTime parse = LocalDateTime.parse(value, dateTimeFormatter); - addValueToRow(rowIndex, col, parse); - } - break; - case STRING: - checkArgument( - mt.equals(Types.MinorType.VARCHAR), - "seaTunnel type is %1$s, but arrow type is %2$s.", - currentType, - mt.name()); - VarCharVector varCharVector = (VarCharVector) curFieldVector; - for (int rowIndex = 0; rowIndex < rowCountInOneBatch; rowIndex++) { - if (varCharVector.isNull(rowIndex)) { - addValueToRow(rowIndex, col, null); - continue; - } - String value = new String(varCharVector.get(rowIndex)); - addValueToRow(rowIndex, col, value); - } - break; - default: - throw new StarRocksConnectorException( - StarRocksConnectorErrorCode.READER_ARROW_DATA_FAILED, - String.format( - "Unsupported type %s", - seaTunnelDataTypes[col].getSqlType().name())); - } - } - } catch (Exception e) { - close(); - throw new StarRocksConnectorException( - StarRocksConnectorErrorCode.READER_ARROW_DATA_FAILED, e); - } - } - - public SeaTunnelRow next() { - if (!hasNext()) { - throw new StarRocksConnectorException( - StarRocksConnectorErrorCode.READER_ARROW_DATA_FAILED, - String.format( - "Get row offset: %d larger than row size: %d", - offsetInRowBatch, readRowCount)); - } - return seaTunnelRowBatch.get(offsetInRowBatch++); - } - - public int getReadRowCount() { - return readRowCount; - } - - public void close() { - try { - if (arrowStreamReader != null) { - arrowStreamReader.close(); - } - if (rootAllocator != null) { - rootAllocator.close(); - } - } catch (IOException e) { - throw new StarRocksConnectorException( - StarRocksConnectorErrorCode.READER_ARROW_DATA_FAILED, - "Failed to close ArrowStreamReader", - e); - } - } -} diff --git a/seatunnel-shade/pom.xml b/seatunnel-shade/pom.xml index 2b378739de7..0011f6b37d2 100644 --- a/seatunnel-shade/pom.xml +++ b/seatunnel-shade/pom.xml @@ -30,13 +30,12 @@ seatunnel-hadoop3-3.1.4-uber seatunnel-jackson seatunnel-guava - seatunnel-arrow-5.0 seatunnel-thrift-service seatunnel-hazelcast seatunnel-janino seatunnel-jetty9-9.4.56 seatunnel-hadoop-aws - + seatunnel-arrow diff --git a/seatunnel-shade/seatunnel-arrow-5.0/pom.xml b/seatunnel-shade/seatunnel-arrow/pom.xml similarity index 78% rename from seatunnel-shade/seatunnel-arrow-5.0/pom.xml rename to seatunnel-shade/seatunnel-arrow/pom.xml index f0d1f2b32b5..27b49ef731c 100644 --- a/seatunnel-shade/seatunnel-arrow-5.0/pom.xml +++ b/seatunnel-shade/seatunnel-arrow/pom.xml @@ -22,11 +22,11 @@ ${revision} - seatunnel-arrow-5.0 - SeaTunnel : Shade : Arrow5.0 + seatunnel-arrow + SeaTunnel : Shade : Arrow - 5.0.0 + 15.0.1 @@ -39,36 +39,12 @@ org.apache.arrow arrow-memory-netty ${arrow.version} - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - io.netty - netty-common - - - - - io.netty - netty-common - 4.1.77.Final - org.apache.maven.plugins maven-shade-plugin @@ -77,8 +53,8 @@ package - seatunnel-arrow-5.0 - ${enableSourceJarCreation} + seatunnel-arrow + true true false false @@ -97,13 +73,17 @@ org.apache.arrow ${seatunnel.shade.package}.org.apache.arrow + + io.netty + ${seatunnel.shade.package}.io.netty + com.google.flatbuffers ${seatunnel.shade.package}.com.google.flatbuffers - io.netty - ${seatunnel.shade.package}.io.netty + com.fasterxml.jackson + ${seatunnel.shade.package}.com.fasterxml.jackson @@ -123,7 +103,7 @@ - ${basedir}/target/seatunnel-arrow-5.0.jar + ${basedir}/target/seatunnel-arrow.jar jar optional @@ -134,5 +114,4 @@ - diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index b2012936bd2..62924093d1b 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -70,3 +70,13 @@ jetty-util-9.4.56.v20240826.jar jetty-util-ajax-9.4.56.v20240826.jar javax.servlet-api-3.1.0.jar seatunnel-jetty9-9.4.56-2.3.9-SNAPSHOT-optional.jar +arrow-format-15.0.1.jar +arrow-memory-core-15.0.1.jar +arrow-memory-netty-15.0.1.jar +arrow-vector-15.0.1.jar +eclipse-collections-11.1.0.jar +eclipse-collections-api-11.1.0.jar +flatbuffers-java-23.5.26.jar +netty-buffer-4.1.104.Final.jar +netty-common-4.1.104.Final.jar +seatunnel-arrow-2.3.9-SNAPSHOT-optional.jar From 47164379c0ec54f8c1121b77a4f66d1c6d53ceb2 Mon Sep 17 00:00:00 2001 From: zhangdonghao Date: Tue, 3 Dec 2024 15:33:20 +0800 Subject: [PATCH 2/3] [Feature][core] support arrow transfers data to SeatunnelRow in arrow format [Feature][core] support arrow transfers data to SeatunnelRow in arrow format --- .../arrow/converter/BigIntConverter.java | 33 ----- .../source/arrow/converter/BitConverter.java | 33 ----- .../source/arrow/converter/Converter.java | 8 +- .../arrow/converter/DateDayConverter.java | 33 ----- .../arrow/converter/DateMilliConverter.java | 33 ----- .../arrow/converter/Decimal256Converter.java | 33 ----- .../arrow/converter/DecimalConverter.java | 33 ----- ...t4Converter.java => DefaultConverter.java} | 9 +- .../arrow/converter/DenseUnionConverter.java | 33 ----- .../arrow/converter/DurationConverter.java | 33 ----- .../converter/ExtensionTypeConverter.java | 33 ----- .../converter/FixedSizeBinaryConverter.java | 33 ----- .../converter/FixedSizeListConverter.java | 8 +- .../arrow/converter/Float4Converter.java | 33 ----- .../arrow/converter/Float8Converter.java | 33 ----- .../source/arrow/converter/IntConverter.java | 33 ----- .../arrow/converter/IntervalDayConverter.java | 33 ----- .../IntervalMonthDayNanoConverter.java | 33 ----- .../converter/IntervalYearConverter.java | 33 ----- .../arrow/converter/LargeListConverter.java | 8 +- .../converter/LargeVarBinaryConverter.java | 33 ----- .../converter/LargeVarCharConverter.java | 33 ----- .../source/arrow/converter/ListConverter.java | 9 +- .../source/arrow/converter/MapConverter.java | 11 +- .../arrow/converter/SmallIntConverter.java | 34 ----- .../arrow/converter/StructConverter.java | 28 ++++ .../arrow/converter/TimeMicroConverter.java | 33 ----- .../arrow/converter/TimeMilliConverter.java | 33 ----- .../arrow/converter/TimeNanoConverter.java | 33 ----- .../arrow/converter/TimeSecConverter.java | 33 ----- .../converter/TimeStampMicroTZConverter.java | 33 ----- .../converter/TimeStampMilliTZConverter.java | 33 ----- .../converter/TimeStampNanoTZConverter.java | 33 ----- .../converter/TimeStampSecConverter.java | 33 ----- .../converter/TimeStampSecTZConverter.java | 33 ----- .../arrow/converter/TinyIntConverter.java | 34 ----- .../arrow/converter/UInt1Converter.java | 33 ----- .../arrow/converter/UInt2Converter.java | 33 ----- .../arrow/converter/UInt8Converter.java | 33 ----- .../arrow/converter/UnionConverter.java | 33 ----- .../arrow/converter/VarBinaryConverter.java | 33 ----- .../arrow/converter/VarCharConverter.java | 33 ----- ...er.java => ArrowToSeatunnelRowReader.java} | 124 ++++++++++++------ ...el.common.source.arrow.converter.Converter | 36 ----- ...ava => ArrowToSeatunnelRowReaderTest.java} | 13 +- .../doris/source/reader/DorisValueReader.java | 13 +- .../client/source/StarRocksBeReadClient.java | 6 +- 47 files changed, 161 insertions(+), 1269 deletions(-) delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BigIntConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BitConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateDayConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateMilliConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Decimal256Converter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DecimalConverter.java rename seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/{UInt4Converter.java => DefaultConverter.java} (82%) delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DenseUnionConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DurationConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ExtensionTypeConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeBinaryConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float4Converter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float8Converter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalDayConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalMonthDayNanoConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalYearConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarBinaryConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarCharConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/SmallIntConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMicroConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMilliConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeNanoConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeSecConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroTZConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliTZConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoTZConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecTZConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TinyIntConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt1Converter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt2Converter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt8Converter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UnionConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarBinaryConverter.java delete mode 100644 seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarCharConverter.java rename seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/{Arrow2SeatunnelRowReader.java => ArrowToSeatunnelRowReader.java} (74%) rename seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/{Arrow2SeatunnelRowReaderTest.java => ArrowToSeatunnelRowReaderTest.java} (98%) diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BigIntConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BigIntConverter.java deleted file mode 100644 index 0b64895e44c..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BigIntConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.BigIntVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class BigIntConverter implements Converter { - @Override - public Object convert(int rowIndex, BigIntVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.BIGINT == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BitConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BitConverter.java deleted file mode 100644 index b099816642a..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/BitConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.BitVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class BitConverter implements Converter { - @Override - public Object convert(int rowIndex, BitVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.BIT == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Converter.java index 0fdcc08349f..4c367e14ae7 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Converter.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Converter.java @@ -20,14 +20,18 @@ import org.apache.seatunnel.shade.org.apache.arrow.vector.FieldVector; import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; -import java.util.List; +import java.util.Map; import java.util.function.Function; public interface Converter { + String ARRAY_KEY = "ARRAY"; + String MAP_KEY = "KEY"; + String MAP_VALUE = "VALUE"; + Object convert(int rowIndex, T fieldVector); - default Object convert(int rowIndex, T fieldVector, List genericsConvert) { + default Object convert(int rowIndex, T fieldVector, Map genericsConverters) { throw new UnsupportedOperationException("Unsupported generics convert"); } diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateDayConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateDayConverter.java deleted file mode 100644 index d62c0d239ee..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateDayConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.DateDayVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class DateDayConverter implements Converter { - @Override - public Object convert(int rowIndex, DateDayVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.DATEDAY == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateMilliConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateMilliConverter.java deleted file mode 100644 index 429c0fae3c3..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DateMilliConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.DateMilliVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class DateMilliConverter implements Converter { - @Override - public Object convert(int rowIndex, DateMilliVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.DATEMILLI == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Decimal256Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Decimal256Converter.java deleted file mode 100644 index 362a4b70446..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Decimal256Converter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.Decimal256Vector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class Decimal256Converter implements Converter { - @Override - public Object convert(int rowIndex, Decimal256Vector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.DECIMAL256 == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DecimalConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DecimalConverter.java deleted file mode 100644 index 17c5e255c09..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DecimalConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.DecimalVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class DecimalConverter implements Converter { - @Override - public Object convert(int rowIndex, DecimalVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.DECIMAL == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt4Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DefaultConverter.java similarity index 82% rename from seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt4Converter.java rename to seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DefaultConverter.java index d34cfcc7218..7a9b6d0a47e 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt4Converter.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DefaultConverter.java @@ -17,17 +17,18 @@ package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; -import org.apache.seatunnel.shade.org.apache.arrow.vector.UInt4Vector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.FieldVector; import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; -public class UInt4Converter implements Converter { +public class DefaultConverter implements Converter { + @Override - public Object convert(int rowIndex, UInt4Vector fieldVector) { + public Object convert(int rowIndex, FieldVector fieldVector) { return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); } @Override public boolean support(Types.MinorType type) { - return Types.MinorType.UINT4 == type; + return false; } } diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DenseUnionConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DenseUnionConverter.java deleted file mode 100644 index c60521248d3..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DenseUnionConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.DenseUnionVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class DenseUnionConverter implements Converter { - @Override - public Object convert(int rowIndex, DenseUnionVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.DENSEUNION == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DurationConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DurationConverter.java deleted file mode 100644 index 090cec6fce5..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/DurationConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.DurationVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class DurationConverter implements Converter { - @Override - public Object convert(int rowIndex, DurationVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.DURATION == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ExtensionTypeConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ExtensionTypeConverter.java deleted file mode 100644 index 7571b2186c5..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ExtensionTypeConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.ExtensionTypeVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class ExtensionTypeConverter implements Converter { - @Override - public Object convert(int rowIndex, ExtensionTypeVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.EXTENSIONTYPE == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeBinaryConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeBinaryConverter.java deleted file mode 100644 index e91e8a6d108..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeBinaryConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.FixedSizeBinaryVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class FixedSizeBinaryConverter implements Converter { - @Override - public Object convert(int rowIndex, FixedSizeBinaryVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.FIXEDSIZEBINARY == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java index d36ad321f7b..7dcc9b30eb4 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java @@ -24,6 +24,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -35,11 +36,12 @@ public Object convert(int rowIndex, FixedSizeListVector fieldVector) { @Override public Object convert( - int rowIndex, FixedSizeListVector fieldVector, List genericsConvert) { + int rowIndex, FixedSizeListVector fieldVector, Map genericsConverters) { if (fieldVector.isNull(rowIndex)) { return null; } List listData = fieldVector.getObject(rowIndex); + Function converter = genericsConverters.get(ARRAY_KEY); return listData.stream() .map( item -> { @@ -49,9 +51,9 @@ public Object convert( .atZone(ZoneOffset.UTC) .withZoneSameInstant(ZoneId.systemDefault()) .toLocalDateTime(); - return genericsConvert.get(0).apply(localDateTime); + return converter.apply(localDateTime); } else { - return genericsConvert.get(0).apply(item); + return converter.apply(item); } }) .collect(Collectors.toList()); diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float4Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float4Converter.java deleted file mode 100644 index e45873ed7c3..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float4Converter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.Float4Vector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class Float4Converter implements Converter { - @Override - public Object convert(int rowIndex, Float4Vector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.FLOAT4 == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float8Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float8Converter.java deleted file mode 100644 index d5322e099a0..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/Float8Converter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.Float8Vector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class Float8Converter implements Converter { - @Override - public Object convert(int rowIndex, Float8Vector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.FLOAT8 == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntConverter.java deleted file mode 100644 index 77135a8f4ef..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.IntVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class IntConverter implements Converter { - @Override - public Object convert(int rowIndex, IntVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.INT == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalDayConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalDayConverter.java deleted file mode 100644 index 815ae6a4b66..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalDayConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.IntervalDayVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class IntervalDayConverter implements Converter { - @Override - public Object convert(int rowIndex, IntervalDayVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.INTERVALDAY == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalMonthDayNanoConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalMonthDayNanoConverter.java deleted file mode 100644 index 9afe312c9aa..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalMonthDayNanoConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.IntervalMonthDayNanoVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class IntervalMonthDayNanoConverter implements Converter { - @Override - public Object convert(int rowIndex, IntervalMonthDayNanoVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.INTERVALMONTHDAYNANO == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalYearConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalYearConverter.java deleted file mode 100644 index 6f80f8e79cd..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/IntervalYearConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.IntervalYearVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class IntervalYearConverter implements Converter { - @Override - public Object convert(int rowIndex, IntervalYearVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.INTERVALYEAR == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeListConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeListConverter.java index fe3763a09e0..f42921f951b 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeListConverter.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeListConverter.java @@ -24,6 +24,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -35,11 +36,12 @@ public Object convert(int rowIndex, LargeListVector fieldVector) { @Override public Object convert( - int rowIndex, LargeListVector fieldVector, List genericsConvert) { + int rowIndex, LargeListVector fieldVector, Map genericsConverters) { if (fieldVector.isNull(rowIndex)) { return null; } List listData = fieldVector.getObject(rowIndex); + Function converter = genericsConverters.get(ARRAY_KEY); return listData.stream() .map( item -> { @@ -49,9 +51,9 @@ public Object convert( .atZone(ZoneOffset.UTC) .withZoneSameInstant(ZoneId.systemDefault()) .toLocalDateTime(); - return genericsConvert.get(0).apply(localDateTime); + return converter.apply(localDateTime); } else { - return genericsConvert.get(0).apply(item); + return converter.apply(item); } }) .collect(Collectors.toList()); diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarBinaryConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarBinaryConverter.java deleted file mode 100644 index fa9a42132a1..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarBinaryConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.LargeVarBinaryVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class LargeVarBinaryConverter implements Converter { - @Override - public Object convert(int rowIndex, LargeVarBinaryVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.LARGEVARBINARY == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarCharConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarCharConverter.java deleted file mode 100644 index 6b0d90a638f..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/LargeVarCharConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.LargeVarCharVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class LargeVarCharConverter implements Converter { - @Override - public Object convert(int rowIndex, LargeVarCharVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex).toString(); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.LARGEVARCHAR == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ListConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ListConverter.java index 48bf0b722b4..647d0497ac8 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ListConverter.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/ListConverter.java @@ -24,6 +24,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; @@ -34,11 +35,13 @@ public Object convert(int rowIndex, ListVector fieldVector) { } @Override - public Object convert(int rowIndex, ListVector fieldVector, List genericsConvert) { + public Object convert( + int rowIndex, ListVector fieldVector, Map genericsConverters) { if (fieldVector.isNull(rowIndex)) { return null; } List listData = fieldVector.getObject(rowIndex); + Function converter = genericsConverters.get(ARRAY_KEY); return listData.stream() .map( item -> { @@ -48,9 +51,9 @@ public Object convert(int rowIndex, ListVector fieldVector, List gener .atZone(ZoneOffset.UTC) .withZoneSameInstant(ZoneId.systemDefault()) .toLocalDateTime(); - return genericsConvert.get(0).apply(localDateTime); + return converter.apply(localDateTime); } else { - return genericsConvert.get(0).apply(item); + return converter.apply(item); } }) .collect(Collectors.toList()); diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/MapConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/MapConverter.java index 9c66d9ac936..18ec3c6910e 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/MapConverter.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/MapConverter.java @@ -25,7 +25,6 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.function.Function; @@ -36,14 +35,16 @@ public Object convert(int rowIndex, MapVector fieldVector) { } @Override - public Object convert(int rowIndex, MapVector fieldVector, List genericsConvert) { + public Object convert( + int rowIndex, MapVector fieldVector, Map genericsConverters) { UnionMapReader reader = fieldVector.getReader(); reader.setPosition(rowIndex); Map mapValue = new HashMap<>(); + Function keyConverter = genericsConverters.get(MAP_KEY); + Function valueConverter = genericsConverters.get(MAP_VALUE); while (reader.next()) { - Object key = genericsConvert.get(0).apply(processTimeZone(reader.key().readObject())); - Object value = - genericsConvert.get(1).apply(processTimeZone(reader.value().readObject())); + Object key = keyConverter.apply(processTimeZone(reader.key().readObject())); + Object value = valueConverter.apply(processTimeZone(reader.value().readObject())); mapValue.put(key, value); } return mapValue; diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/SmallIntConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/SmallIntConverter.java deleted file mode 100644 index d634892c66d..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/SmallIntConverter.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.SmallIntVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class SmallIntConverter implements Converter { - - @Override - public Object convert(int rowIndex, SmallIntVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.SMALLINT == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/StructConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/StructConverter.java index d45c58ad2f6..b74ccde554b 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/StructConverter.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/StructConverter.java @@ -20,12 +20,40 @@ import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.StructVector; import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j public class StructConverter implements Converter { @Override public Object convert(int rowIndex, StructVector fieldVector) { return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); } + @Override + public Object convert( + int rowIndex, StructVector fieldVector, Map genericsConverters) { + Map valueMap = fieldVector.getObject(rowIndex); + return valueMap.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> { + Optional optional = + Optional.ofNullable(genericsConverters.get(e.getKey())); + if (optional.isPresent()) { + return optional.get().apply(e.getValue()); + } else { + log.warn("No converter found for key:{}", e.getKey()); + return e.getValue(); + } + })); + } + @Override public boolean support(Types.MinorType type) { return Types.MinorType.STRUCT == type; diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMicroConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMicroConverter.java deleted file mode 100644 index 2e4786a04a5..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMicroConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeMicroVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class TimeMicroConverter implements Converter { - @Override - public Object convert(int rowIndex, TimeMicroVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.TIMEMICRO == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMilliConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMilliConverter.java deleted file mode 100644 index 56a2c0d54bc..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeMilliConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeMilliVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class TimeMilliConverter implements Converter { - @Override - public Object convert(int rowIndex, TimeMilliVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.TIMEMILLI == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeNanoConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeNanoConverter.java deleted file mode 100644 index 8cabd2eaf02..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeNanoConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeNanoVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class TimeNanoConverter implements Converter { - @Override - public Object convert(int rowIndex, TimeNanoVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.TIMENANO == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeSecConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeSecConverter.java deleted file mode 100644 index 6d4615d7327..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeSecConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeSecVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class TimeSecConverter implements Converter { - @Override - public Object convert(int rowIndex, TimeSecVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.TIMESEC == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroTZConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroTZConverter.java deleted file mode 100644 index 4eb774f1217..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMicroTZConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMicroTZVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class TimeStampMicroTZConverter implements Converter { - @Override - public Object convert(int rowIndex, TimeStampMicroTZVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.TIMESTAMPMICROTZ == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliTZConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliTZConverter.java deleted file mode 100644 index 11652819afe..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampMilliTZConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampMilliTZVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class TimeStampMilliTZConverter implements Converter { - @Override - public Object convert(int rowIndex, TimeStampMilliTZVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.TIMESTAMPMILLITZ == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoTZConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoTZConverter.java deleted file mode 100644 index bd68a3ba94a..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampNanoTZConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampNanoTZVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class TimeStampNanoTZConverter implements Converter { - @Override - public Object convert(int rowIndex, TimeStampNanoTZVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.TIMESTAMPNANOTZ == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java deleted file mode 100644 index 461880fda37..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampSecVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class TimeStampSecConverter implements Converter { - @Override - public Object convert(int rowIndex, TimeStampSecVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.TIMESTAMPSEC == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecTZConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecTZConverter.java deleted file mode 100644 index ee0e23ab18f..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TimeStampSecTZConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.TimeStampSecTZVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class TimeStampSecTZConverter implements Converter { - @Override - public Object convert(int rowIndex, TimeStampSecTZVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.TIMESTAMPSECTZ == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TinyIntConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TinyIntConverter.java deleted file mode 100644 index 717cd3fccb7..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/TinyIntConverter.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.TinyIntVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class TinyIntConverter implements Converter { - - @Override - public Object convert(int rowIndex, TinyIntVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.TINYINT == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt1Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt1Converter.java deleted file mode 100644 index a85d13d9918..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt1Converter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.UInt1Vector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class UInt1Converter implements Converter { - @Override - public Object convert(int rowIndex, UInt1Vector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.UINT1 == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt2Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt2Converter.java deleted file mode 100644 index 9717b2da9f3..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt2Converter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.UInt2Vector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class UInt2Converter implements Converter { - @Override - public Object convert(int rowIndex, UInt2Vector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.UINT2 == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt8Converter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt8Converter.java deleted file mode 100644 index 390924f4bd4..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UInt8Converter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.UInt8Vector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class UInt8Converter implements Converter { - @Override - public Object convert(int rowIndex, UInt8Vector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.UINT8 == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UnionConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UnionConverter.java deleted file mode 100644 index 5b3b0a953f7..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/UnionConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.complex.UnionVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class UnionConverter implements Converter { - @Override - public Object convert(int rowIndex, UnionVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.UNION == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarBinaryConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarBinaryConverter.java deleted file mode 100644 index 1f850cce060..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarBinaryConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.VarBinaryVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class VarBinaryConverter implements Converter { - @Override - public Object convert(int rowIndex, VarBinaryVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.VARBINARY == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarCharConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarCharConverter.java deleted file mode 100644 index 5efd54278fa..00000000000 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/VarCharConverter.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter; - -import org.apache.seatunnel.shade.org.apache.arrow.vector.VarCharVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; - -public class VarCharConverter implements Converter { - @Override - public Object convert(int rowIndex, VarCharVector fieldVector) { - return fieldVector.isNull(rowIndex) ? null : fieldVector.getObject(rowIndex).toString(); - } - - @Override - public boolean support(Types.MinorType type) { - return Types.MinorType.VARCHAR == type; - } -} diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/Arrow2SeatunnelRowReader.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java similarity index 74% rename from seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/Arrow2SeatunnelRowReader.java rename to seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java index a3cb35ded98..8a5c4f575aa 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/Arrow2SeatunnelRowReader.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java @@ -17,13 +17,7 @@ package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader; -import org.apache.seatunnel.shade.org.apache.arrow.memory.RootAllocator; -import org.apache.seatunnel.shade.org.apache.arrow.vector.FieldVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.seatunnel.shade.org.apache.arrow.vector.ipc.ArrowStreamReader; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; -import org.apache.seatunnel.shade.org.apache.arrow.vector.util.Text; - +import lombok.extern.slf4j.Slf4j; import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.MapType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; @@ -31,8 +25,13 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.api.table.type.SqlType; import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter; - -import lombok.extern.slf4j.Slf4j; +import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DefaultConverter; +import org.apache.seatunnel.shade.org.apache.arrow.memory.RootAllocator; +import org.apache.seatunnel.shade.org.apache.arrow.vector.FieldVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.seatunnel.shade.org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; +import org.apache.seatunnel.shade.org.apache.arrow.vector.util.Text; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -44,8 +43,6 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -53,7 +50,7 @@ import java.util.function.Function; @Slf4j -public class Arrow2SeatunnelRowReader implements AutoCloseable { +public class ArrowToSeatunnelRowReader implements AutoCloseable { private final SeaTunnelDataType[] seaTunnelDataTypes; private int offsetInRowBatch = 0; @@ -61,29 +58,40 @@ public class Arrow2SeatunnelRowReader implements AutoCloseable { private int readRowCount = 0; private List fieldVectors; private VectorSchemaRoot root; - private final ArrowStreamReader arrowStreamReader; - private final RootAllocator rootAllocator; + private ArrowStreamReader arrowStreamReader; + private RootAllocator rootAllocator; private final Map fieldIndexMap = new HashMap<>(); private final List seatunnelRowBatch = new ArrayList<>(); - private final List converters = new ArrayList<>(); + private static final List converters = new ArrayList<>(); + private final DefaultConverter defaultConverter = new DefaultConverter(); private final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); private final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("HH:mm:ss"); private final DateTimeFormatter DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); - public Arrow2SeatunnelRowReader(byte[] byteArray, SeaTunnelRowType seaTunnelRowType) { + static { + ServiceLoader.load(Converter.class).forEach(converters::add); + } + + public ArrowToSeatunnelRowReader(byte[] byteArray, SeaTunnelRowType seaTunnelRowType) { this.seaTunnelDataTypes = seaTunnelRowType.getFieldTypes(); - this.rootAllocator = new RootAllocator(Integer.MAX_VALUE); - this.arrowStreamReader = - new ArrowStreamReader(new ByteArrayInputStream(byteArray), rootAllocator); + initFieldIndexMap(seaTunnelRowType); + initArrowReader(byteArray); + } + + private void initFieldIndexMap(SeaTunnelRowType seaTunnelRowType) { for (int i = 0; i < seaTunnelRowType.getFieldNames().length; i++) { fieldIndexMap.put(seaTunnelRowType.getFieldNames()[i], i); } - ServiceLoader load = ServiceLoader.load(Converter.class); - load.forEach(converters::add); } - public Arrow2SeatunnelRowReader readArrow() { + private void initArrowReader(byte[] byteArray) { + this.rootAllocator = new RootAllocator(Integer.MAX_VALUE); + this.arrowStreamReader = + new ArrowStreamReader(new ByteArrayInputStream(byteArray), rootAllocator); + } + + public ArrowToSeatunnelRowReader readArrow() { try { this.root = arrowStreamReader.getVectorSchemaRoot(); while (arrowStreamReader.loadNextBatch()) { @@ -214,31 +222,65 @@ private Object convertArrowData( Types.MinorType minorType, FieldVector fieldVector, SeaTunnelDataType seaTunnelDataType) { + if (seaTunnelDataType == null) { + throw new IllegalArgumentException("seaTunnelDataType cannot be null"); + } + for (Converter converter : converters) { if (converter.support(minorType)) { SqlType sqlType = seaTunnelDataType.getSqlType(); - if (SqlType.MAP == sqlType) { - MapType mapType = (MapType) seaTunnelDataType; - SqlType keyType = mapType.getKeyType().getSqlType(); - SqlType valueType = mapType.getValueType().getSqlType(); - return converter.convert( - rowIndex, - fieldVector, - Arrays.asList(genericsConvert(keyType), genericsConvert(valueType))); - } else if (SqlType.ARRAY == sqlType) { - ArrayType arrayType = (ArrayType) seaTunnelDataType; - SqlType elementType = arrayType.getElementType().getSqlType(); - return converter.convert( - rowIndex, - fieldVector, - Collections.singletonList(genericsConvert(elementType))); - } else { - return converter.convert(rowIndex, fieldVector); + switch (sqlType) { + case MAP: + return convertMap( + rowIndex, converter, fieldVector, (MapType) seaTunnelDataType); + case ARRAY: + return convertArray( + rowIndex, converter, fieldVector, (ArrayType) seaTunnelDataType); + case ROW: + return convertRow( + rowIndex, + converter, + fieldVector, + (SeaTunnelRowType) seaTunnelDataType); + default: + return converter.convert(rowIndex, fieldVector); } } } - log.error("No converter found for Arrow MinorType: {}", minorType); - throw new IllegalStateException("Unsupported Arrow MinorType: " + minorType); + return defaultConverter.convert(rowIndex, fieldVector); + } + + private Object convertMap( + int rowIndex, Converter converter, FieldVector fieldVector, MapType mapType) { + SqlType keyType = mapType.getKeyType().getSqlType(); + SqlType valueType = mapType.getValueType().getSqlType(); + Map fieldConverters = new HashMap<>(); + fieldConverters.put(Converter.MAP_KEY, genericsConvert(keyType)); + fieldConverters.put(Converter.MAP_VALUE, genericsConvert(valueType)); + return converter.convert( + rowIndex, + fieldVector, + fieldConverters); + } + + private Object convertArray( + int rowIndex, Converter converter, FieldVector fieldVector, ArrayType arrayType) { + SqlType elementType = arrayType.getElementType().getSqlType(); + Map fieldConverters = new HashMap<>(); + fieldConverters.put(Converter.ARRAY_KEY, genericsConvert(elementType)); + return converter.convert( + rowIndex, fieldVector, fieldConverters); + } + + private Object convertRow( + int rowIndex, Converter converter, FieldVector fieldVector, SeaTunnelRowType rowType) { + String[] fieldNames = rowType.getFieldNames(); + List> fieldTypes = rowType.getChildren(); + Map fieldConverters = new HashMap<>(); + for (int i = 0; i < fieldTypes.size(); i++) { + fieldConverters.put(fieldNames[i], genericsConvert(fieldTypes.get(i).getSqlType())); + } + return converter.convert(rowIndex, fieldVector, fieldConverters); } private Function genericsConvert(SqlType sqlType) { diff --git a/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter b/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter index c804b406495..dfdefc680ed 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter +++ b/seatunnel-connectors-v2/connector-common/src/main/resources/META-INF/services/org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter @@ -13,48 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.BigIntConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.BitConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DateDayConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DateMilliConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Decimal256Converter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DecimalConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DenseUnionConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DurationConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.ExtensionTypeConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.FixedSizeBinaryConverter org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.FixedSizeListConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Float4Converter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Float8Converter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.IntConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.IntervalDayConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.IntervalMonthDayNanoConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.IntervalYearConverter org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.LargeListConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.LargeVarBinaryConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.LargeVarCharConverter org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.ListConverter org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.MapConverter org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.NullConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.SmallIntConverter org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.StructConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeMicroConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeMilliConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeNanoConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeSecConverter org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMicroConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMicroTZConverter org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMilliConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampMilliTZConverter org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampNanoConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampNanoTZConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampSecConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TimeStampSecTZConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.TinyIntConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.UInt1Converter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.UInt2Converter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.UInt4Converter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.UInt8Converter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.UnionConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.VarBinaryConverter -org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.VarCharConverter diff --git a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/Arrow2SeatunnelRowReaderTest.java b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java similarity index 98% rename from seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/Arrow2SeatunnelRowReaderTest.java rename to seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java index 9603e1afc10..8c85a4f1070 100644 --- a/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/Arrow2SeatunnelRowReaderTest.java +++ b/seatunnel-connectors-v2/connector-common/src/test/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/ArrowToSeatunnelRowReaderTest.java @@ -54,7 +54,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader.Arrow2SeatunnelRowReader; +import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader.ArrowToSeatunnelRowReader; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -78,7 +78,7 @@ import java.util.stream.Collectors; @Slf4j -public class Arrow2SeatunnelRowReaderTest { +public class ArrowToSeatunnelRowReaderTest { private static VectorSchemaRoot root; private static RootAllocator rootAllocator; @@ -305,8 +305,8 @@ public void testSeatunnelRow() throws Exception { writer.writeBatch(); out.flush(); List rows = new ArrayList<>(); - try (Arrow2SeatunnelRowReader reader = - new Arrow2SeatunnelRowReader(out.toByteArray(), getSeatunnelRowType(true)) + try (ArrowToSeatunnelRowReader reader = + new ArrowToSeatunnelRowReader(out.toByteArray(), getSeatunnelRowType(true)) .readArrow()) { while (reader.hasNext()) { rows.add(reader.next()); @@ -419,8 +419,9 @@ public void testConvertArrowSpeed() throws Exception { List rows = new ArrayList<>(); stopwatch.reset().start(); SeaTunnelRowType seatunnelRowType = getSeatunnelRowType(false); - try (Arrow2SeatunnelRowReader reader = - new Arrow2SeatunnelRowReader(out.toByteArray(), seatunnelRowType).readArrow()) { + try (ArrowToSeatunnelRowReader reader = + new ArrowToSeatunnelRowReader(out.toByteArray(), seatunnelRowType) + .readArrow()) { while (reader.hasNext()) { rows.add(reader.next()); } diff --git a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/reader/DorisValueReader.java b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/reader/DorisValueReader.java index deb9b585252..e68366cf260 100644 --- a/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/reader/DorisValueReader.java +++ b/seatunnel-connectors-v2/connector-doris/src/main/java/org/apache/seatunnel/connectors/doris/source/reader/DorisValueReader.java @@ -28,7 +28,7 @@ import org.apache.seatunnel.connectors.doris.source.DorisSourceTable; import org.apache.seatunnel.connectors.doris.source.serialization.Routing; import org.apache.seatunnel.connectors.doris.util.SchemaUtils; -import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader.Arrow2SeatunnelRowReader; +import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader.ArrowToSeatunnelRowReader; import org.apache.doris.sdk.thrift.TScanBatchResult; import org.apache.doris.sdk.thrift.TScanCloseParams; @@ -60,12 +60,12 @@ public class DorisValueReader { protected int offset = 0; protected AtomicBoolean eos = new AtomicBoolean(false); - protected Arrow2SeatunnelRowReader rowBatch; + protected ArrowToSeatunnelRowReader rowBatch; // flag indicate if support deserialize Arrow to RowBatch asynchronously protected boolean deserializeArrowToRowBatchAsync; - protected BlockingQueue rowBatchBlockingQueue; + protected BlockingQueue rowBatchBlockingQueue; private TScanOpenParams openParams; protected String contextId; protected Schema schema; @@ -158,8 +158,8 @@ public void run() { TScanBatchResult nextResult = client.getNext(nextBatchParams); eos.set(nextResult.isEos()); if (!eos.get()) { - Arrow2SeatunnelRowReader rowBatch = - new Arrow2SeatunnelRowReader( + ArrowToSeatunnelRowReader rowBatch = + new ArrowToSeatunnelRowReader( nextResult.getRows(), seaTunnelRowType) .readArrow(); @@ -236,7 +236,8 @@ public boolean hasNext() { eos.set(nextResult.isEos()); if (!eos.get()) { rowBatch = - new Arrow2SeatunnelRowReader(nextResult.getRows(), seaTunnelRowType) + new ArrowToSeatunnelRowReader( + nextResult.getRows(), seaTunnelRowType) .readArrow(); } } diff --git a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksBeReadClient.java b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksBeReadClient.java index 6a1e90dfa0c..c0be0106bb0 100644 --- a/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksBeReadClient.java +++ b/seatunnel-connectors-v2/connector-starrocks/src/main/java/org/apache/seatunnel/connectors/seatunnel/starrocks/client/source/StarRocksBeReadClient.java @@ -19,7 +19,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader.Arrow2SeatunnelRowReader; +import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader.ArrowToSeatunnelRowReader; import org.apache.seatunnel.connectors.seatunnel.starrocks.client.source.model.QueryPartition; import org.apache.seatunnel.connectors.seatunnel.starrocks.config.SourceConfig; import org.apache.seatunnel.connectors.seatunnel.starrocks.exception.StarRocksConnectorErrorCode; @@ -57,7 +57,7 @@ public class StarRocksBeReadClient implements Serializable { private int readerOffset = 0; private final SourceConfig sourceConfig; private SeaTunnelRowType seaTunnelRowType; - private Arrow2SeatunnelRowReader rowBatch; + private ArrowToSeatunnelRowReader rowBatch; protected AtomicBoolean eos = new AtomicBoolean(false); public StarRocksBeReadClient(String beNodeInfo, SourceConfig sourceConfig) { @@ -165,7 +165,7 @@ public boolean hasNext() { if (!eos.get()) { rowBatch = - new Arrow2SeatunnelRowReader(result.getRows(), seaTunnelRowType) + new ArrowToSeatunnelRowReader(result.getRows(), seaTunnelRowType) .readArrow(); } } catch (TException e) { From 890ef6f23eaf0a5cbcee9dacdb0be8bc59d7948c Mon Sep 17 00:00:00 2001 From: zhangdonghao Date: Tue, 3 Dec 2024 20:04:26 +0800 Subject: [PATCH 3/3] [Feature][core] support arrow transfers data to SeatunnelRow in arrow format --- .../converter/FixedSizeListConverter.java | 4 +++- .../reader/ArrowToSeatunnelRowReader.java | 24 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java index 7dcc9b30eb4..c325cc50cde 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/converter/FixedSizeListConverter.java @@ -36,7 +36,9 @@ public Object convert(int rowIndex, FixedSizeListVector fieldVector) { @Override public Object convert( - int rowIndex, FixedSizeListVector fieldVector, Map genericsConverters) { + int rowIndex, + FixedSizeListVector fieldVector, + Map genericsConverters) { if (fieldVector.isNull(rowIndex)) { return null; } diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java index 8a5c4f575aa..6bd23bad0bc 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/source/arrow/reader/ArrowToSeatunnelRowReader.java @@ -17,7 +17,13 @@ package org.apache.seatunnel.connectors.seatunnel.common.source.arrow.reader; -import lombok.extern.slf4j.Slf4j; +import org.apache.seatunnel.shade.org.apache.arrow.memory.RootAllocator; +import org.apache.seatunnel.shade.org.apache.arrow.vector.FieldVector; +import org.apache.seatunnel.shade.org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.seatunnel.shade.org.apache.arrow.vector.ipc.ArrowStreamReader; +import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; +import org.apache.seatunnel.shade.org.apache.arrow.vector.util.Text; + import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.MapType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; @@ -26,12 +32,8 @@ import org.apache.seatunnel.api.table.type.SqlType; import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.Converter; import org.apache.seatunnel.connectors.seatunnel.common.source.arrow.converter.DefaultConverter; -import org.apache.seatunnel.shade.org.apache.arrow.memory.RootAllocator; -import org.apache.seatunnel.shade.org.apache.arrow.vector.FieldVector; -import org.apache.seatunnel.shade.org.apache.arrow.vector.VectorSchemaRoot; -import org.apache.seatunnel.shade.org.apache.arrow.vector.ipc.ArrowStreamReader; -import org.apache.seatunnel.shade.org.apache.arrow.vector.types.Types; -import org.apache.seatunnel.shade.org.apache.arrow.vector.util.Text; + +import lombok.extern.slf4j.Slf4j; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -257,10 +259,7 @@ private Object convertMap( Map fieldConverters = new HashMap<>(); fieldConverters.put(Converter.MAP_KEY, genericsConvert(keyType)); fieldConverters.put(Converter.MAP_VALUE, genericsConvert(valueType)); - return converter.convert( - rowIndex, - fieldVector, - fieldConverters); + return converter.convert(rowIndex, fieldVector, fieldConverters); } private Object convertArray( @@ -268,8 +267,7 @@ private Object convertArray( SqlType elementType = arrayType.getElementType().getSqlType(); Map fieldConverters = new HashMap<>(); fieldConverters.put(Converter.ARRAY_KEY, genericsConvert(elementType)); - return converter.convert( - rowIndex, fieldVector, fieldConverters); + return converter.convert(rowIndex, fieldVector, fieldConverters); } private Object convertRow(