Merge branch 'main' into binary_literal

koralium · Jan 24, 2025 · 3f46f04 · 3f46f04
2 parents 6c147fb + 3c24ab2
commit 3f46f04
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 0 deletions.
diff --git a/src/FlowtideDotNet.Core/ColumnStore/TreeStorage/ArrowToInternalVisitor.cs b/src/FlowtideDotNet.Core/ColumnStore/TreeStorage/ArrowToInternalVisitor.cs
@@ -200,6 +200,8 @@ public void Visit(DenseUnionArray array)
                 }
 
                 columns.Add(_dataColumn ?? throw new InvalidOperationException("Internal column is null"));
+                // Reset null counter
+                _nullCount = 0;
             }
 
             _dataColumn = new UnionColumn(columns, typeMemory, offsetMemory, array.Length, preAllocatedMemoryManager);

diff --git a/tests/FlowtideDotNet.Core.Tests/ColumnStore/ArrowTests.cs b/tests/FlowtideDotNet.Core.Tests/ColumnStore/ArrowTests.cs
@@ -378,6 +378,39 @@ public void UnionSerializeDeserialize()
             Assert.Equal(1, deserializedBatch.Columns[0].GetValueAt(2, default).AsLong);
             Assert.Equal("2", deserializedBatch.Columns[0].GetValueAt(3, default).ToString());
             Assert.Equal(2, deserializedBatch.Columns[0].GetValueAt(4, default).AsLong);
+
+
+        }
+
+        [Fact]
+        public void UnionSerializeDeserializeWithOnlyNull()
+        {
+            Column column = new Column(GlobalMemoryManager.Instance);
+            column.Add(new StringValue("1"));
+            column.Add(NullValue.Instance);
+            column.Add(new Int64Value(1));
+
+            Column toInsertInto = new Column(GlobalMemoryManager.Instance);
+
+            toInsertInto.InsertRangeFrom(0, column, 1, 1);
+
+            var recordBatch = EventArrowSerializer.BatchToArrow(new EventBatchData(
+            [
+                toInsertInto
+            ]), toInsertInto.Count);
+
+            MemoryStream memoryStream = new MemoryStream();
+            var writer = new ArrowStreamWriter(memoryStream, recordBatch.Schema, true);
+            writer.WriteRecordBatch(recordBatch);
+            writer.Dispose();
+            memoryStream.Position = 0;
+            var reader = new ArrowStreamReader(memoryStream, new Apache.Arrow.Memory.NativeMemoryAllocator(), true);
+            var deserializedRecordBatch = reader.ReadNextRecordBatch();
+            var deserializedBatch = EventArrowSerializer.ArrowToBatch(deserializedRecordBatch, GlobalMemoryManager.Instance);
+
+            // This threw an error since null count was incorrectly set on the union column from the null column inside
+            deserializedBatch.Columns[0].RemoveAt(0);
+            Assert.Empty(deserializedBatch);
         }
 
         [Fact]