Skip to content

Commit

Permalink
Merge branch 'master' into SNOW-1675321-remove-account-identifier-que…
Browse files Browse the repository at this point in the history 
…stion
  • Loading branch information
@sfc-gh-wfateem
sfc-gh-wfateem authored Oct 22, 2024
2 parents 8b26c79 + 90b7007 commit 5e1b061
Show file tree
Hide file tree
Showing 34 changed files with 1,783 additions and 556 deletions.
Binary file modified .github/workflows/parameters/parameters_AWS.json.gpg
View file
Binary file not shown.
3 changes: 2 additions & 1 deletion .github/workflows/parameters/parameters_AZURE.json.gpg
View file
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
� V���&|���@US��ҹ�'j��M�*,#঍�Q^���s(Q��Z��Nl��,3�x���GZ�&�����L] �64+@2-�E����F�ȇ����#~����m�JQti)��k���f/g�B����Xd6�6��=fdM�R�%� ��*�?�����d������G~���'u�8��Px��(B�ib��D4"g�V�� ;��mU�+� %��6UPJuw �H'�9�v�㰅WB} 9�� ���A�"{ é�g)��� �Ysa�
� P�&�V�X����S}jV�;�0�����D��ssx(G���� 0�n�P�UŴ‰Ob�p�w3cH��i�6�W�<ﾽ�U�^�K1x���TbiAmvu2��.+ʝ���.��f*����\��V�RY/ti����}�?!�@�E���r��}�c,..p��f���k��Ny�������I�[��;K�?d�iv�x&8X%NW+,���[�>3���`������� ��H���s��cFI���
}�v�ź�m�l��c�����"A���4��
Expand Down
Binary file modified .github/workflows/parameters/parameters_GCP.json.gpg
View file
Binary file not shown.
8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,14 @@ Snowflake data types and their .NET types is covered in: [Data Types and Data Fo
How execute a query, use query bindings, run queries synchronously and asynchronously:
[Running Queries and Reading Results](doc/QueryingData.md)

## Structured types

Using structured types: [Structured types](doc/StructuredTypes.md)

## Vector type

Using vector type: [Vector type](doc/VectorType.md)

## Stage Files

Using stage files within PUT/GET commands:
Expand Down
133 changes: 95 additions & 38 deletions Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
*/
#nullable enable

using System;
using System.Data;
Expand Down Expand Up @@ -87,7 +88,7 @@ public void TestBindNullValue()
foreach (DbType type in Enum.GetValues(typeof(DbType)))
{
bool isTypeSupported = true;
string colName = null;
string colName;
using (IDbCommand command = dbConnection.CreateCommand())
{
var param = command.CreateParameter();
Expand Down Expand Up @@ -226,7 +227,7 @@ public void TestBindValue()
foreach (DbType type in Enum.GetValues(typeof(DbType)))
{
bool isTypeSupported = true;
string colName = null;
string colName;
using (IDbCommand command = dbConnection.CreateCommand())
{
var param = command.CreateParameter();
Expand Down Expand Up @@ -361,7 +362,7 @@ public void TestBindValueWithSFDataType()
dbConnection.Open();
foreach (SFDataType type in Enum.GetValues(typeof(SFDataType)))
{
if (!type.Equals(SFDataType.None) && !type.Equals(SFDataType.MAP))
if (!type.Equals(SFDataType.None) && !type.Equals(SFDataType.MAP) && !type.Equals(SFDataType.VECTOR))
{
bool isTypeSupported = true;
string[] columns;
Expand Down Expand Up @@ -649,6 +650,7 @@ public void TestPutArrayBind()
var count = cmd.ExecuteNonQuery();
Assert.AreEqual(total * 3, count);

cmd.Parameters.Clear();
cmd.CommandText = $"SELECT * FROM {TableName}";
IDataReader reader = cmd.ExecuteReader();
Assert.IsTrue(reader.Read());
Expand Down Expand Up @@ -884,13 +886,20 @@ public void TestExplicitDbTypeAssignmentForArrayValue()
[TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
*/
// Session TimeZone cases
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType, SFDataType columnType, Int32? columnPrecision, DbType bindingType, string comparisonFormat, string timeZone)
{
// Arrange
var timestamp = "2023/03/15 13:17:29.207 +05:00"; // 08:17:29.207 UTC
var expected = ExpectedTimestampWrapper.From(timestamp, columnType);
string[] timestamps =
{
"2023/03/15 13:17:29.207 +05:00",
"9999/12/30 23:24:25.987 +07:00",
"0001/01/02 02:06:07.000 -04:00"
};
var expected = ExpectedTimestampWrapper.From(timestamps, columnType);
var columnWithPrecision = ColumnTypeWithPrecision(columnType, columnPrecision);
var testCase = $"ResultFormat={resultFormat}, TableType={tableType}, ColumnType={columnWithPrecision}, BindingType={bindingType}, ComparisonFormat={comparisonFormat}";
var bindingThreshold = 65280; // when exceeded enforces bindings via file on stage
Expand All @@ -906,24 +915,34 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
if (!timeZone.IsNullOrEmpty()) // Driver ignores this setting and relies on local environment timezone
conn.ExecuteNonQuery($"alter session set TIMEZONE = '{timeZone}'");

// prepare initial column
var columns = new List<String> { "id number(10,0) not null primary key" };
var sql_columns = "id";
var sql_values = "?";

// prepare additional columns
for (int i = 1; i <= timestamps.Length; ++i)
{
columns.Add($"ts_{i} {columnWithPrecision}");
sql_columns += $",ts_{i}";
sql_values += ",?";
}

CreateOrReplaceTable(conn,
TableName,
tableType.TableDDLCreationPrefix(),
new[] {
"id number(10,0) not null primary key", // necessary only for HYBRID tables
$"ts {columnWithPrecision}"
},
columns,
tableType.TableDDLCreationFlags());

// Act+Assert
var sqlInsert = $"insert into {TableName} (id, ts) values (?, ?)";
var sqlInsert = $"insert into {TableName} ({sql_columns}) values ({sql_values})";
InsertSingleRecord(conn, sqlInsert, bindingType, 1, expected);
InsertMultipleRecords(conn, sqlInsert, bindingType, 2, expected, smallBatchRowCount, false);
InsertMultipleRecords(conn, sqlInsert, bindingType, smallBatchRowCount+2, expected, bigBatchRowCount, true);

// Assert
var row = 0;
using (var select = conn.CreateCommand($"select id, ts from {TableName} order by id"))
using (var select = conn.CreateCommand($"select {sql_columns} from {TableName} order by id"))
{
s_logger.Debug(select.CommandText);
var reader = select.ExecuteReader();
Expand All @@ -932,7 +951,11 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
++row;
string faultMessage = $"Mismatch for row: {row}, {testCase}";
Assert.AreEqual(row, reader.GetInt32(0));
expected.AssertEqual(reader.GetValue(1), comparisonFormat, faultMessage);

for (int i = 0; i < timestamps.Length; ++i)
{
expected.AssertEqual(reader.GetValue(i + 1), comparisonFormat, faultMessage, i);
}
}
}
Assert.AreEqual(1+smallBatchRowCount+bigBatchRowCount, row);
Expand All @@ -947,12 +970,24 @@ private void InsertSingleRecord(IDbConnection conn, string sqlInsert, DbType bin
insert.Add("1", DbType.Int32, identifier);
if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
{
var parameter = (SnowflakeDbParameter)insert.Add("2", binding, ts.GetDateTimeOffset());
parameter.SFDataType = ts.ExpectedColumnType();
var dateTimeOffsets = ts.GetDateTimeOffsets();
for (int i = 0; i < dateTimeOffsets.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = dateTimeOffsets[i];
var parameter = insert.Add(parameterName, binding, parameterValue);
parameter.SFDataType = ts.ExpectedColumnType();
}
}
else
{
insert.Add("2", binding, ts.GetDateTime());
var dateTimes = ts.GetDateTimes();
for (int i = 0; i < dateTimes.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = dateTimes[i];
insert.Add(parameterName, binding, parameterValue);
}
}

// Act
Expand All @@ -973,12 +1008,25 @@ private void InsertMultipleRecords(IDbConnection conn, string sqlInsert, DbType
insert.Add("1", DbType.Int32, Enumerable.Range(initialIdentifier, rowsCount).ToArray());
if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
{
var parameter = (SnowflakeDbParameter)insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTimeOffset(), rowsCount).ToArray());
parameter.SFDataType = ts.ExpectedColumnType();
var dateTimeOffsets = ts.GetDateTimeOffsets();
for (int i = 0; i < dateTimeOffsets.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = Enumerable.Repeat(dateTimeOffsets[i], rowsCount).ToArray();
var parameter = insert.Add(parameterName, binding, parameterValue);
parameter.SFDataType = ts.ExpectedColumnType();
}

}
else
{
insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTime(), rowsCount).ToArray());
var dateTimes = ts.GetDateTimes();
for (int i = 0; i < dateTimes.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = Enumerable.Repeat(dateTimes[i], rowsCount).ToArray();
insert.Add(parameterName, binding, parameterValue);
}
}

// Act
Expand All @@ -1001,57 +1049,66 @@ private static string ColumnTypeWithPrecision(SFDataType columnType, Int32? colu
class ExpectedTimestampWrapper
{
private readonly SFDataType _columnType;
private readonly DateTime? _expectedDateTime;
private readonly DateTimeOffset? _expectedDateTimeOffset;
private readonly DateTime[]? _expectedDateTimes;
private readonly DateTimeOffset[]? _expectedDateTimeOffsets;

internal static ExpectedTimestampWrapper From(string timestampWithTimeZone, SFDataType columnType)
internal static ExpectedTimestampWrapper From(string[] timestampsWithTimeZone, SFDataType columnType)
{
if (IsOffsetType(columnType))
{
var dateTimeOffset = DateTimeOffset.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
return new ExpectedTimestampWrapper(dateTimeOffset, columnType);
var dateTimeOffsets =
timestampsWithTimeZone
.Select(ts => DateTimeOffset.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture))
.ToArray();
return new ExpectedTimestampWrapper(dateTimeOffsets, columnType);
}

var dateTime = DateTime.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
return new ExpectedTimestampWrapper(dateTime, columnType);
var dateTimes =
timestampsWithTimeZone
.Select(ts => DateTime.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture))
.ToArray();

return new ExpectedTimestampWrapper(dateTimes, columnType);
}

private ExpectedTimestampWrapper(DateTime dateTime, SFDataType columnType)
private ExpectedTimestampWrapper(DateTime[] dateTimes, SFDataType columnType)
{
_expectedDateTime = dateTime;
_expectedDateTimeOffset = null;
_expectedDateTimes = dateTimes;
_expectedDateTimeOffsets = null;
_columnType = columnType;
}

private ExpectedTimestampWrapper(DateTimeOffset dateTimeOffset, SFDataType columnType)
private ExpectedTimestampWrapper(DateTimeOffset[] dateTimeOffsets, SFDataType columnType)
{
_expectedDateTimeOffset = dateTimeOffset;
_expectedDateTime = null;
_expectedDateTimeOffsets = dateTimeOffsets;
_expectedDateTimes = null;
_columnType = columnType;
}

internal SFDataType ExpectedColumnType() => _columnType;

internal void AssertEqual(object actual, string comparisonFormat, string faultMessage)
internal void AssertEqual(object actual, string comparisonFormat, string faultMessage, int index)
{
switch (_columnType)
{
case SFDataType.TIMESTAMP_TZ:
Assert.AreEqual(GetDateTimeOffset().ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimeOffsets()[index].ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
break;
case SFDataType.TIMESTAMP_LTZ:
Assert.AreEqual(GetDateTimeOffset().ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimeOffsets()[index].ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
break;
default:
Assert.AreEqual(GetDateTime().ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimes()[index].ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
break;
}
}

internal DateTime GetDateTime() => _expectedDateTime ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
internal DateTime[] GetDateTimes() => _expectedDateTimes ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");

internal DateTimeOffset GetDateTimeOffset() => _expectedDateTimeOffset ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
internal DateTimeOffset[] GetDateTimeOffsets() => _expectedDateTimeOffsets ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");

internal static bool IsOffsetType(SFDataType type) => type == SFDataType.TIMESTAMP_LTZ || type == SFDataType.TIMESTAMP_TZ;
}
}

#nullable restore
Loading

0 comments on commit 5e1b061

Please sign in to comment.