SNOW-1731124 Arrow batches fields

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/BigIntVectorConverter.java

@@ -8,6 +8,8 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.BigIntVector;
 import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class BigIntVectorConverter extends SimpleArrowFullVectorConverter<BigIntVector> {
@@ -21,14 +23,20 @@ public BigIntVectorConverter(
     super(allocator, vector, context, session, idx);
   }
 
+  private static FieldType getFieldType(boolean nullable) {
+    return new FieldType(nullable, new ArrowType.Int(64, true), null);
+  }
+
   @Override
   protected boolean matchingType() {
     return (vector instanceof BigIntVector);
   }
 
   @Override
   protected BigIntVector initVector() {
-    BigIntVector resultVector = new BigIntVector(vector.getName(), allocator);
+    boolean nullable = vector.getField().isNullable();
+    BigIntVector resultVector =
+        new BigIntVector(vector.getName(), getFieldType(nullable), allocator);
     resultVector.allocateNew(vector.getValueCount());
     return resultVector;
   }

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/BinaryVectorConverter.java

@@ -8,6 +8,8 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.VarBinaryVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class BinaryVectorConverter extends SimpleArrowFullVectorConverter<VarBinaryVector> {
@@ -20,14 +22,20 @@ public BinaryVectorConverter(
     super(allocator, vector, context, session, idx);
   }
 
+  private static FieldType getFieldType(boolean nullable) {
+    return new FieldType(nullable, new ArrowType.Binary(), null);
+  }
+
   @Override
   protected boolean matchingType() {
     return vector instanceof VarBinaryVector;
   }
 
   @Override
   protected VarBinaryVector initVector() {
-    VarBinaryVector resultVector = new VarBinaryVector(vector.getName(), allocator);
+    boolean nullable = vector.getField().isNullable();
+    VarBinaryVector resultVector =
+        new VarBinaryVector(vector.getName(), getFieldType(nullable), allocator);
     resultVector.allocateNew(vector.getValueCount());
     return resultVector;
   }

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/BitVectorConverter.java

@@ -8,6 +8,8 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.BitVector;
 import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class BitVectorConverter extends SimpleArrowFullVectorConverter<BitVector> {
@@ -21,14 +23,19 @@ public BitVectorConverter(
     super(allocator, vector, context, session, idx);
   }
 
+  private static FieldType getFieldType(boolean nullable) {
+    return new FieldType(nullable, new ArrowType.Bool(), null);
+  }
+
   @Override
   protected boolean matchingType() {
     return vector instanceof BitVector;
   }
 
   @Override
   protected BitVector initVector() {
-    BitVector resultVector = new BitVector(vector.getName(), allocator);
+    boolean nullable = vector.getField().isNullable();
+    BitVector resultVector = new BitVector(vector.getName(), getFieldType(nullable), allocator);
     resultVector.allocateNew(vector.getValueCount());
     return resultVector;
   }

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/DateVectorConverter.java

@@ -9,6 +9,9 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.DateDayVector;
 import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.DateUnit;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class DateVectorConverter extends SimpleArrowFullVectorConverter<DateDayVector> {
@@ -25,14 +28,20 @@ public DateVectorConverter(
     this.timeZone = timeZone;
   }
 
+  private static FieldType getFieldType(boolean nullable) {
+    return new FieldType(nullable, new ArrowType.Date(DateUnit.DAY), null);
+  }
+
   @Override
   protected boolean matchingType() {
     return vector instanceof DateDayVector;
   }
 
   @Override
   protected DateDayVector initVector() {
-    DateDayVector resultVector = new DateDayVector(vector.getName(), allocator);
+    boolean nullable = vector.getField().isNullable();
+    DateDayVector resultVector =
+        new DateDayVector(vector.getName(), getFieldType(nullable), allocator);
     resultVector.allocateNew(vector.getValueCount());
     return resultVector;
   }

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/DecimalVectorConverter.java

@@ -8,6 +8,8 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.DecimalVector;
 import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class DecimalVectorConverter extends SimpleArrowFullVectorConverter<DecimalVector> {
@@ -21,6 +23,10 @@ public DecimalVectorConverter(
     super(allocator, vector, context, session, idx);
   }
 
+  private static FieldType getFieldType(boolean nullable, int scale, int precision) {
+    return new FieldType(nullable, new ArrowType.Decimal(precision, scale, 128), null);
+  }
+
   @Override
   protected boolean matchingType() {
     return (vector instanceof DecimalVector);
@@ -32,7 +38,9 @@ protected DecimalVector initVector() {
     String precisionString = vector.getField().getMetadata().get("precision");
     int scale = Integer.parseInt(scaleString);
     int precision = Integer.parseInt(precisionString);
-    DecimalVector resultVector = new DecimalVector(vector.getName(), allocator, precision, scale);
+    boolean nullable = vector.getField().isNullable();
+    DecimalVector resultVector =
+        new DecimalVector(vector.getName(), getFieldType(nullable, scale, precision), allocator);
     resultVector.allocateNew(vector.getValueCount());
     return resultVector;
   }

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/FixedSizeListVectorConverter.java

@@ -12,7 +12,9 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.FixedSizeListVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class FixedSizeListVectorConverter extends AbstractFullVectorConverter {
@@ -41,6 +43,10 @@ public class FixedSizeListVectorConverter extends AbstractFullVectorConverter {
     this.valueTargetType = valueTargetType;
   }
 
+  private static FieldType getFieldType(boolean nullable, int size) {
+    return new FieldType(nullable, new ArrowType.FixedSizeList(size), null);
+  }
+
   @Override
   protected FieldVector convertVector()
       throws SFException, SnowflakeSQLException, SFArrowException {
@@ -50,8 +56,12 @@ protected FieldVector convertVector()
       FieldVector convertedDataVector =
           ArrowFullVectorConverterUtil.convert(
               allocator, dataVector, context, session, timeZoneToUse, 0, valueTargetType);
+
+      boolean nullable = vector.getField().isNullable();
+      int listSize = listVector.getListSize();
       FixedSizeListVector convertedListVector =
-          FixedSizeListVector.empty(listVector.getName(), listVector.getListSize(), allocator);
+          new FixedSizeListVector(
+              listVector.getName(), allocator, getFieldType(nullable, listSize), null);
       ArrayList<Field> fields = new ArrayList<>();
       fields.add(convertedDataVector.getField());
       convertedListVector.initializeChildrenFromFields(fields);

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/FloatVectorConverter.java

@@ -8,6 +8,9 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.Float8Vector;
 import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class FloatVectorConverter extends SimpleArrowFullVectorConverter<Float8Vector> {
@@ -26,9 +29,16 @@ protected boolean matchingType() {
     return vector instanceof Float8Vector;
   }
 
+  private static FieldType getFieldType(boolean nullable) {
+    return new FieldType(
+        nullable, new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE), null);
+  }
+
   @Override
   protected Float8Vector initVector() {
-    Float8Vector resultVector = new Float8Vector(vector.getName(), allocator);
+    boolean nullable = vector.getField().isNullable();
+    Float8Vector resultVector =
+        new Float8Vector(vector.getName(), getFieldType(nullable), allocator);
     resultVector.allocateNew(vector.getValueCount());
     return resultVector;
   }

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/IntVectorConverter.java

@@ -8,6 +8,8 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class IntVectorConverter extends SimpleArrowFullVectorConverter<IntVector> {
@@ -26,9 +28,14 @@ protected boolean matchingType() {
     return (vector instanceof IntVector);
   }
 
+  private static FieldType getFieldType(boolean nullable) {
+    return new FieldType(nullable, new ArrowType.Int(32, true), null);
+  }
+
   @Override
   protected IntVector initVector() {
-    IntVector resultVector = new IntVector(vector.getName(), allocator);
+    boolean nullable = vector.getField().isNullable();
+    IntVector resultVector = new IntVector(vector.getName(), getFieldType(nullable), allocator);
     resultVector.allocateNew(vector.getValueCount());
     return resultVector;
   }

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/ListVectorConverter.java

@@ -12,7 +12,9 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class ListVectorConverter extends AbstractFullVectorConverter {
@@ -41,8 +43,13 @@ public class ListVectorConverter extends AbstractFullVectorConverter {
     this.valueTargetType = valueTargetType;
   }
 
+  private static FieldType getFieldType(boolean nullable) {
+    return new FieldType(nullable, new ArrowType.List(), null);
+  }
+
   protected ListVector initVector(String name, Field field) {
-    ListVector convertedListVector = ListVector.empty(name, allocator);
+    boolean nullable = vector.getField().isNullable();
+    ListVector convertedListVector = new ListVector(name, allocator, getFieldType(nullable), null);
     ArrayList<Field> fields = new ArrayList<>();
     fields.add(field);
     convertedListVector.initializeChildrenFromFields(fields);
@@ -58,11 +65,13 @@ protected FieldVector convertVector()
       FieldVector convertedDataVector =
           ArrowFullVectorConverterUtil.convert(
               allocator, dataVector, context, session, timeZoneToUse, 0, valueTargetType);
-      // TODO: change to convertedDataVector and make all necessary changes to make it work
-      ListVector convertedListVector = initVector(vector.getName(), dataVector.getField());
+      ListVector convertedListVector = initVector(vector.getName(), convertedDataVector.getField());
       convertedListVector.allocateNew();
       convertedListVector.setValueCount(listVector.getValueCount());
-      convertedListVector.getOffsetBuffer().setBytes(0, listVector.getOffsetBuffer());
+
+      ArrowBuf offsetBuffer = listVector.getOffsetBuffer();
+      convertedListVector.getOffsetBuffer().setBytes(0L, offsetBuffer, 0L, offsetBuffer.capacity());
+
       ArrowBuf validityBuffer = listVector.getValidityBuffer();
       convertedListVector
           .getValidityBuffer()

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/MapVectorConverter.java

@@ -9,7 +9,9 @@
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class MapVectorConverter extends ListVectorConverter {
@@ -25,9 +27,15 @@ public class MapVectorConverter extends ListVectorConverter {
     super(allocator, vector, context, session, timeZoneToUse, idx, valueTargetType);
   }
 
+  private static FieldType getFieldType(boolean nullable) {
+    return new FieldType(nullable, new ArrowType.Map(false), null);
+  }
+
   @Override
   protected ListVector initVector(String name, Field field) {
-    MapVector convertedMapVector = MapVector.empty(name, allocator, false);
+    boolean nullable = vector.getField().isNullable();
+    MapVector convertedMapVector =
+        new MapVector(vector.getName(), allocator, getFieldType(nullable), null);
     ArrayList<Field> fields = new ArrayList<>();
     fields.add(field);
     convertedMapVector.initializeChildrenFromFields(fields);

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/SmallIntVectorConverter.java

@@ -8,6 +8,8 @@
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.SmallIntVector;
 import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.FieldType;
 
 @SnowflakeJdbcInternalApi
 public class SmallIntVectorConverter extends SimpleArrowFullVectorConverter<SmallIntVector> {
@@ -21,14 +23,20 @@ public SmallIntVectorConverter(
     super(allocator, vector, context, session, idx);
   }
 
+  private static FieldType getFieldType(boolean nullable) {
+    return new FieldType(nullable, new ArrowType.Int(16, true), null);
+  }
+
   @Override
   protected boolean matchingType() {
     return (vector instanceof SmallIntVector);
   }
 
   @Override
   protected SmallIntVector initVector() {
-    SmallIntVector resultVector = new SmallIntVector(vector.getName(), allocator);
+    boolean nullable = vector.getField().isNullable();
+    SmallIntVector resultVector =
+        new SmallIntVector(vector.getName(), getFieldType(nullable), allocator);
     resultVector.allocateNew(vector.getValueCount());
     return resultVector;
   }

src/main/java/net/snowflake/client/core/arrow/fullvectorconverters/StructVectorConverter.java

@@ -15,7 +15,9 @@
 import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.complex.StructVector;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.TransferPair;
 
 @SnowflakeJdbcInternalApi
@@ -45,6 +47,10 @@ public class StructVectorConverter extends AbstractFullVectorConverter {
     this.targetTypes = targetTypes;
   }
 
+  private static FieldType getFieldType(boolean nullable) {
+    return new FieldType(nullable, new ArrowType.Struct(), null);
+  }
+
   protected FieldVector convertVector()
       throws SFException, SnowflakeSQLException, SFArrowException {
     try {
@@ -63,7 +69,10 @@ protected FieldVector convertVector()
 
       List<Field> convertedFields =
           convertedVectors.stream().map(ValueVector::getField).collect(Collectors.toList());
-      StructVector converted = StructVector.empty(vector.getName(), allocator);
+
+      boolean nullable = vector.getField().isNullable();
+      StructVector converted =
+          new StructVector(vector.getName(), allocator, getFieldType(nullable), null);
       converted.allocateNew();
       converted.initializeChildrenFromFields(convertedFields);
       for (FieldVector convertedVector : convertedVectors) {