Skip to content

Commit

Permalink
Start moving the reader to data chunks Api
Browse files Browse the repository at this point in the history
  • Loading branch information
@Giorgi
Giorgi committed Sep 5, 2023
1 parent 05f211e commit 649a925
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 30 deletions.
22 changes: 22 additions & 0 deletions DuckDB.NET.Bindings/DuckDBNativeObjects.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,4 +232,26 @@ private static DuckDBInterval FromTimeSpan(TimeSpan timeSpan)
, Convert.ToUInt64(timeSpan.Ticks / 10 - new TimeSpan(timeSpan.Days, 0, 0, 0).Ticks / 10)
);
}

[StructLayout(LayoutKind.Explicit)]
public struct DuckDBString
{
[FieldOffset(0)] public DuckDBStringPointer Pointer;
[FieldOffset(0)] public DuckDBStringInlined Inlined;
}

[StructLayout(LayoutKind.Explicit)]
public struct DuckDBStringPointer
{
[FieldOffset(0)] public uint length;
[FieldOffset(4)] public IntPtr prefix;
[FieldOffset(8)] public IntPtr ptr;
}

[StructLayout(LayoutKind.Explicit)]
public struct DuckDBStringInlined
{
[FieldOffset(0)] public uint length;
[FieldOffset(4)] public IntPtr inlined;
}
}
36 changes: 36 additions & 0 deletions DuckDB.NET.Bindings/NativeMethods.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,36 @@ public static class Query
public static extern IntPtr DuckDBResultError([In, Out] DuckDBResult result);
}

public static class DataChunks
{
[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_data_chunk_get_column_count")]
public static extern long DuckDBDataChunkGetColumnCount(IntPtr chunk);

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_data_chunk_get_vector")]
public static extern IntPtr DuckDBDataChunkGetVector(IntPtr chunk, long columnIndex);

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_data_chunk_get_size")]
public static extern long DuckDBDataChunkGetSize(IntPtr chunk);

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_vector_get_column_type")]
public static extern IntPtr DuckDBVectorGetColumnType(IntPtr vector);

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_vector_get_data")]
public static extern IntPtr DuckDBVectorGetData(IntPtr vector);

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_vector_get_validity")]
public static extern UIntPtr DuckDBVectorGetValidity(IntPtr vector);

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_list_vector_get_child")]
public static extern long DuckDBListVectorGetChild(IntPtr vector);

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_list_vector_get_size")]
public static extern long DuckDBListVectorGetSize(IntPtr vector);

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_struct_vector_get_child")]
public static extern long DuckDBStructVectorGetChild(IntPtr vector, long index);
}

public static class Types
{
[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_value_boolean")]
Expand Down Expand Up @@ -139,6 +169,12 @@ public static class Types

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_value_timestamp")]
public static extern DuckDBTimestampStruct DuckDBValueTimestamp([In, Out] DuckDBResult result, long col, long row);

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_result_get_chunk")]
public static extern IntPtr DuckDBResultGetChunk(DuckDBResult result, long chunkIndex);

[DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_result_chunk_count")]
public static extern long DuckDBResultChunkCount(DuckDBResult result);
}

public static class PreparedStatements
Expand Down
23 changes: 13 additions & 10 deletions DuckDB.NET.Bindings/Utils.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,34 +13,37 @@ public static bool IsSuccess(this DuckDBState duckDBState)
return duckDBState == DuckDBState.DuckDBSuccess;
}

public static string ToManagedString(this IntPtr unmanagedString, bool freeWhenCopied = true)
public static string ToManagedString(this IntPtr unmanagedString, bool freeWhenCopied = true, int? length = null)
{
string result;
#if NET6_0_OR_GREATER
result = Marshal.PtrToStringUTF8(unmanagedString);
result = length.HasValue? Marshal.PtrToStringUTF8(unmanagedString, length.Value) : Marshal.PtrToStringUTF8(unmanagedString);
#else
if (unmanagedString == IntPtr.Zero)
{
return "";
}

var length = 0;

while (Marshal.ReadByte(unmanagedString, length) != 0)
if (length == null)
{
length++;
length = 0;

while (Marshal.ReadByte(unmanagedString, length.Value) != 0)
{
length++;
}
}

if (length == 0)
{
return string.Empty;
}

var byteArray = new byte[length];
var byteArray = new byte[length.Value];

Marshal.Copy(unmanagedString, byteArray, 0, length);
Marshal.Copy(unmanagedString, byteArray, 0, length.Value);

result = Encoding.UTF8.GetString(byteArray, 0, length);
result = Encoding.UTF8.GetString(byteArray, 0, length.Value);
#endif
if (freeWhenCopied)
{
Expand Down Expand Up @@ -93,4 +96,4 @@ internal static int GetMicrosecond(this TimeOnly timeOnly)
}
#endif
}
}
}
87 changes: 67 additions & 20 deletions DuckDB.NET.Data/DuckDBDataReader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ namespace DuckDB.NET.Data
{
public class DuckDBDataReader : DbDataReader
{
private const int InlineStringMaxLength = 12;
private readonly DuckDbCommand command;
private readonly CommandBehavior behavior;

Expand All @@ -25,6 +26,7 @@ public class DuckDBDataReader : DbDataReader

private int fieldCount;
private int recordsAffected;
private Dictionary<int, IntPtr> vectors = new();

internal DuckDBDataReader(DuckDbCommand command, List<DuckDBResult> queryResults, CommandBehavior behavior)
{
Expand All @@ -38,6 +40,18 @@ internal DuckDBDataReader(DuckDbCommand command, List<DuckDBResult> queryResults

private void InitReaderData()
{
var chunkCount = NativeMethods.Types.DuckDBResultChunkCount(currentResult);
var chunk = NativeMethods.Types.DuckDBResultGetChunk(currentResult, 0);

var columnCount = NativeMethods.DataChunks.DuckDBDataChunkGetColumnCount(chunk);

var size = NativeMethods.DataChunks.DuckDBDataChunkGetSize(chunk);

for (int i = 0; i < columnCount; i++)
{
vectors[i] = NativeMethods.DataChunks.DuckDBDataChunkGetVector(chunk, i);
}

currentRow = -1;
rowCount = NativeMethods.Query.DuckDBRowCount(currentResult);
fieldCount = (int)NativeMethods.Query.DuckDBColumnCount(currentResult);
Expand All @@ -46,17 +60,20 @@ private void InitReaderData()

public override bool GetBoolean(int ordinal)
{
return NativeMethods.Types.DuckDBValueBoolean(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return Marshal.ReadByte(data, currentRow * Marshal.SizeOf<byte>()) != 0;
}

public override byte GetByte(int ordinal)
{
return NativeMethods.Types.DuckDBValueUInt8(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return Marshal.ReadByte(data, currentRow * Marshal.SizeOf<byte>());
}

private sbyte GetSByte(int ordinal)
{
return NativeMethods.Types.DuckDBValueInt8(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return (sbyte)Marshal.ReadByte(data, currentRow * Marshal.SizeOf<byte>());
}

public override long GetBytes(int ordinal, long dataOffset, byte[] buffer, int bufferOffset, int length)
Expand All @@ -81,20 +98,23 @@ public override string GetDataTypeName(int ordinal)

public override DateTime GetDateTime(int ordinal)
{
var timestampStruct = NativeMethods.Types.DuckDBValueTimestamp(currentResult, ordinal, currentRow);

var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
var timestampStruct = Marshal.PtrToStructure<DuckDBTimestampStruct>(data + currentRow * Marshal.SizeOf<DuckDBTimestampStruct>());

return timestampStruct.ToDateTime();
}

private DuckDBDateOnly GetDateOnly(int ordinal)
{
var date = NativeMethods.Types.DuckDBValueDate(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
var date = Marshal.PtrToStructure<DuckDBDate>(data + currentRow * Marshal.SizeOf<DuckDBDate>());
return NativeMethods.DateTime.DuckDBFromDate(date);
}

private DuckDBTimeOnly GetTimeOnly(int ordinal)
{
var time = NativeMethods.Types.DuckDBValueTime(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
var time = Marshal.PtrToStructure<DuckDBTime>(data + currentRow * Marshal.SizeOf<DuckDBTime>());
return NativeMethods.DateTime.DuckDBFromTime(time);
}

Expand All @@ -105,7 +125,8 @@ public override decimal GetDecimal(int ordinal)

public override double GetDouble(int ordinal)
{
return NativeMethods.Types.DuckDBValueDouble(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return Marshal.PtrToStructure<double>(data + currentRow * Marshal.SizeOf<double>());
}

public override Type GetFieldType(int ordinal)
Expand All @@ -132,13 +153,15 @@ public override Type GetFieldType(int ordinal)
DuckDBType.DuckdbTypeVarchar => typeof(string),
DuckDBType.DuckdbTypeDecimal => typeof(decimal),
DuckDBType.DuckdbTypeBlob => typeof(Stream),
var type => throw new ArgumentException($"Unrecognised type {type} ({(int)type}) in column {ordinal + 1}")
var type => throw new ArgumentException(
$"Unrecognised type {type} ({(int)type}) in column {ordinal + 1}")
};
}

public override float GetFloat(int ordinal)
{
return NativeMethods.Types.DuckDBValueFloat(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return Marshal.PtrToStructure<float>(data + currentRow * Marshal.SizeOf<float>());
}

public override Guid GetGuid(int ordinal)
Expand All @@ -148,37 +171,46 @@ public override Guid GetGuid(int ordinal)

public override short GetInt16(int ordinal)
{
return NativeMethods.Types.DuckDBValueInt16(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return Marshal.ReadInt16(data, currentRow * Marshal.SizeOf<short>());
}

public override int GetInt32(int ordinal)
{
return NativeMethods.Types.DuckDBValueInt32(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return Marshal.ReadInt32(data, currentRow * Marshal.SizeOf<int>());
}

public override long GetInt64(int ordinal)
{
return NativeMethods.Types.DuckDBValueInt64(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return Marshal.ReadInt64(data, currentRow * Marshal.SizeOf<long>());
}

private ushort GetUInt16(int ordinal)
{
return NativeMethods.Types.DuckDBValueUInt16(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return (ushort)Marshal.ReadInt32(data, currentRow * Marshal.SizeOf<ushort>());
}

private uint GetUInt32(int ordinal)
{
return NativeMethods.Types.DuckDBValueUInt32(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return (uint)Marshal.ReadInt32(data, currentRow * Marshal.SizeOf<uint>());
}

private ulong GetUInt64(int ordinal)
{
return NativeMethods.Types.DuckDBValueUInt64(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
return (ulong)Marshal.ReadInt32(data, currentRow * Marshal.SizeOf<ulong>());
}

private BigInteger GetBigInteger(int ordinal)
{
return BigInteger.Parse(GetString(ordinal));
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
var hugeInt = Marshal.PtrToStructure<DuckDBHugeInt>(data + currentRow * Marshal.SizeOf<DuckDBHugeInt>());

return hugeInt.ToBigInteger();
}

public override string GetName(int ordinal)
Expand All @@ -203,9 +235,20 @@ public override int GetOrdinal(string name)

public override string GetString(int ordinal)
{
var unmanagedString = NativeMethods.Types.DuckDBValueVarchar(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
data += currentRow * Marshal.SizeOf<DuckDBString>();

var length = Marshal.ReadInt32(data);

return unmanagedString.ToManagedString();
if (length <= InlineStringMaxLength)
{
return (data + Marshal.SizeOf<int>()).ToManagedString(false, length);
}
else
{
var intPtr = Marshal.ReadIntPtr(data + Marshal.SizeOf<int>() * 2);
return intPtr.ToManagedString(false, length);
}
}

public override object GetValue(int ordinal)
Expand Down Expand Up @@ -243,7 +286,9 @@ public override object GetValue(int ordinal)

private DuckDBInterval GetDuckDBInterval(int ordinal)
{
return NativeMethods.Types.DuckDBValueInterval(currentResult, ordinal, currentRow);
var data = NativeMethods.DataChunks.DuckDBVectorGetData(vectors[ordinal]);
var interval = Marshal.PtrToStructure<DuckDBInterval>(data + currentRow * Marshal.SizeOf<DuckDBInterval>());
return interval;
}

public override int GetValues(object[] values)
Expand All @@ -264,6 +309,7 @@ public override Stream GetStream(int ordinal)

public override bool IsDBNull(int ordinal)
{
return false;
var nullMask = NativeMethods.Query.DuckDBNullmaskData(currentResult, ordinal);

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (ubuntu-20.04) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (ubuntu-20.04) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (ubuntu-20.04) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (ubuntu-20.04) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (ubuntu-20.04) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (ubuntu-20.04) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (ubuntu-20.04) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (ubuntu-20.04) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (windows-latest) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (windows-latest) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (windows-latest) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (windows-latest) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (macos-12) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (macos-12) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (macos-12) 

Unreachable code detected

Check warning on line 313 in DuckDB.NET.Data/DuckDBDataReader.cs

View workflow job for this annotation

GitHub Actions / Build library (macos-12) 

Unreachable code detected
return Marshal.ReadByte(nullMask, currentRow) != 0;
}
Expand Down Expand Up @@ -330,6 +376,7 @@ public override DataTable GetSchemaTable()
rowData[4] = true;
table.Rows.Add(rowData);
}

return table;
}

Expand Down

0 comments on commit 649a925

Please sign in to comment.