Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SNOW-1729244 support for large and small timestamps #1038

Merged
merged 9 commits into from
Oct 21, 2024
Merged

SNOW-1729244 support for large and small timestamps #1038

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
93f3cc5
SNOW-1729244 support for large timestamps
sfc-gh-dstempniak Oct 9, 2024
70cba5f
SNOW-1729244 tests for wrong conversion
sfc-gh-dstempniak Oct 9, 2024
6b8bbd4
SNOW-1729244 nullable enabled
sfc-gh-dstempniak Oct 9, 2024
972e90a
SNOW-1729244 fix
sfc-gh-dstempniak Oct 9, 2024
067ecd1
SNOW-1729244 remove warnings
sfc-gh-dstempniak Oct 10, 2024
c8098f0
SNOW-1729244 fixes after CR
sfc-gh-dstempniak Oct 14, 2024
fd737a1
SNOW-1729244 small fixes
sfc-gh-dstempniak Oct 14, 2024
78e4b5c
Merge branch 'master' into SNOW-1729244_large_timestaps
sfc-gh-dstempniak Oct 14, 2024
69d4819
Merge branch 'master' into SNOW-1729244_large_timestaps
sfc-gh-dstempniak Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 91 additions & 37 deletions Snowflake.Data.Tests/IntegrationTests/SFBindTestIT.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/*
* Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
*/
#nullable enable
sfc-gh-knozderko marked this conversation as resolved.
Show resolved Hide resolved

using System;
using System.Data;
Expand Down Expand Up @@ -87,7 +88,7 @@ public void TestBindNullValue()
foreach (DbType type in Enum.GetValues(typeof(DbType)))
{
bool isTypeSupported = true;
string colName = null;
string colName;
using (IDbCommand command = dbConnection.CreateCommand())
{
var param = command.CreateParameter();
Expand Down Expand Up @@ -226,7 +227,7 @@ public void TestBindValue()
foreach (DbType type in Enum.GetValues(typeof(DbType)))
{
bool isTypeSupported = true;
string colName = null;
string colName;
using (IDbCommand command = dbConnection.CreateCommand())
{
var param = command.CreateParameter();
Expand Down Expand Up @@ -884,13 +885,20 @@ public void TestExplicitDbTypeAssignmentForArrayValue()
[TestCase(ResultFormat.ARROW, SFTableType.Iceberg, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, null)]
*/
// Session TimeZone cases
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.JSON, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Europe/Warsaw")]
[TestCase(ResultFormat.ARROW, SFTableType.Standard, SFDataType.TIMESTAMP_LTZ, 6, DbType.DateTimeOffset, FormatYmdHmsZ, "Asia/Tokyo")]
public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType, SFDataType columnType, Int32? columnPrecision, DbType bindingType, string comparisonFormat, string timeZone)
{
// Arrange
var timestamp = "2023/03/15 13:17:29.207 +05:00"; // 08:17:29.207 UTC
var expected = ExpectedTimestampWrapper.From(timestamp, columnType);
string[] timestamps =
{
"2023/03/15 13:17:29.207 +05:00",
"9999/12/30 23:24:25.987 +07:00",
"0001/01/02 02:06:07.000 -04:00"
};
var expected = ExpectedTimestampWrapper.From(timestamps, columnType);
var columnWithPrecision = ColumnTypeWithPrecision(columnType, columnPrecision);
var testCase = $"ResultFormat={resultFormat}, TableType={tableType}, ColumnType={columnWithPrecision}, BindingType={bindingType}, ComparisonFormat={comparisonFormat}";
var bindingThreshold = 65280; // when exceeded enforces bindings via file on stage
Expand All @@ -906,24 +914,34 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
if (!timeZone.IsNullOrEmpty()) // Driver ignores this setting and relies on local environment timezone
conn.ExecuteNonQuery($"alter session set TIMEZONE = '{timeZone}'");

// prepare initial column
var columns = new List<String> { "id number(10,0) not null primary key" };
var sql_columns = "id";
var sql_values = "?";

// prepare additional columns
for (int i = 1; i <= timestamps.Length; ++i)
{
columns.Add($"ts_{i} {columnWithPrecision}");
sql_columns += $",ts_{i}";
sql_values += ",?";
}

CreateOrReplaceTable(conn,
TableName,
tableType.TableDDLCreationPrefix(),
new[] {
"id number(10,0) not null primary key", // necessary only for HYBRID tables
$"ts {columnWithPrecision}"
},
columns,
tableType.TableDDLCreationFlags());

// Act+Assert
var sqlInsert = $"insert into {TableName} (id, ts) values (?, ?)";
var sqlInsert = $"insert into {TableName} ({sql_columns}) values ({sql_values})";
InsertSingleRecord(conn, sqlInsert, bindingType, 1, expected);
InsertMultipleRecords(conn, sqlInsert, bindingType, 2, expected, smallBatchRowCount, false);
InsertMultipleRecords(conn, sqlInsert, bindingType, smallBatchRowCount+2, expected, bigBatchRowCount, true);

// Assert
var row = 0;
using (var select = conn.CreateCommand($"select id, ts from {TableName} order by id"))
using (var select = conn.CreateCommand($"select {sql_columns} from {TableName} order by id"))
{
s_logger.Debug(select.CommandText);
var reader = select.ExecuteReader();
Expand All @@ -932,7 +950,11 @@ public void TestDateTimeBinding(ResultFormat resultFormat, SFTableType tableType
++row;
string faultMessage = $"Mismatch for row: {row}, {testCase}";
Assert.AreEqual(row, reader.GetInt32(0));
expected.AssertEqual(reader.GetValue(1), comparisonFormat, faultMessage);

for (int i = 0; i < timestamps.Length; ++i)
{
expected.AssertEqual(reader.GetValue(i + 1), comparisonFormat, faultMessage, i);
}
}
}
Assert.AreEqual(1+smallBatchRowCount+bigBatchRowCount, row);
Expand All @@ -947,12 +969,24 @@ private void InsertSingleRecord(IDbConnection conn, string sqlInsert, DbType bin
insert.Add("1", DbType.Int32, identifier);
if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
{
var parameter = (SnowflakeDbParameter)insert.Add("2", binding, ts.GetDateTimeOffset());
parameter.SFDataType = ts.ExpectedColumnType();
var dateTimeOffsets = ts.GetDateTimeOffsets();
for (int i = 0; i < dateTimeOffsets.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = dateTimeOffsets[i];
var parameter = insert.Add(parameterName, binding, parameterValue);
parameter.SFDataType = ts.ExpectedColumnType();
}
}
else
{
insert.Add("2", binding, ts.GetDateTime());
var dateTimes = ts.GetDateTimes();
for (int i = 0; i < dateTimes.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = dateTimes[i];
insert.Add(parameterName, binding, parameterValue);
}
}

// Act
Expand All @@ -973,12 +1007,25 @@ private void InsertMultipleRecords(IDbConnection conn, string sqlInsert, DbType
insert.Add("1", DbType.Int32, Enumerable.Range(initialIdentifier, rowsCount).ToArray());
if (ExpectedTimestampWrapper.IsOffsetType(ts.ExpectedColumnType()))
{
var parameter = (SnowflakeDbParameter)insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTimeOffset(), rowsCount).ToArray());
parameter.SFDataType = ts.ExpectedColumnType();
var dateTimeOffsets = ts.GetDateTimeOffsets();
for (int i = 0; i < dateTimeOffsets.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = Enumerable.Repeat(dateTimeOffsets[i], rowsCount).ToArray();
var parameter = insert.Add(parameterName, binding, parameterValue);
parameter.SFDataType = ts.ExpectedColumnType();
}

}
else
{
insert.Add("2", binding, Enumerable.Repeat(ts.GetDateTime(), rowsCount).ToArray());
var dateTimes = ts.GetDateTimes();
for (int i = 0; i < dateTimes.Length; ++i)
{
var parameterName = (i + 2).ToString();
var parameterValue = Enumerable.Repeat(dateTimes[i], rowsCount).ToArray();
insert.Add(parameterName, binding, parameterValue);
}
}

// Act
Expand All @@ -1001,56 +1048,63 @@ private static string ColumnTypeWithPrecision(SFDataType columnType, Int32? colu
class ExpectedTimestampWrapper
{
private readonly SFDataType _columnType;
private readonly DateTime? _expectedDateTime;
private readonly DateTimeOffset? _expectedDateTimeOffset;
private readonly DateTime[]? _expectedDateTimes;
private readonly DateTimeOffset[]? _expectedDateTimeOffsets;

internal static ExpectedTimestampWrapper From(string timestampWithTimeZone, SFDataType columnType)
internal static ExpectedTimestampWrapper From(string[] timestampsWithTimeZone, SFDataType columnType)
{
if (IsOffsetType(columnType))
{
var dateTimeOffset = DateTimeOffset.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
return new ExpectedTimestampWrapper(dateTimeOffset, columnType);
var dateTimeOffsets =
timestampsWithTimeZone
.Select(ts => DateTimeOffset.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture))
.ToArray();
return new ExpectedTimestampWrapper(dateTimeOffsets, columnType);
}

var dateTime = DateTime.ParseExact(timestampWithTimeZone, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture);
return new ExpectedTimestampWrapper(dateTime, columnType);
var dateTimes =
timestampsWithTimeZone
.Select(ts => DateTime.ParseExact(ts, "yyyy/MM/dd HH:mm:ss.fff zzz", CultureInfo.InvariantCulture))
.ToArray();

return new ExpectedTimestampWrapper(dateTimes, columnType);
}

private ExpectedTimestampWrapper(DateTime dateTime, SFDataType columnType)
private ExpectedTimestampWrapper(DateTime[] dateTimes, SFDataType columnType)
{
_expectedDateTime = dateTime;
_expectedDateTimeOffset = null;
_expectedDateTimes = dateTimes;
_expectedDateTimeOffsets = null;
_columnType = columnType;
}

private ExpectedTimestampWrapper(DateTimeOffset dateTimeOffset, SFDataType columnType)
private ExpectedTimestampWrapper(DateTimeOffset[] dateTimeOffsets, SFDataType columnType)
{
_expectedDateTimeOffset = dateTimeOffset;
_expectedDateTime = null;
_expectedDateTimeOffsets = dateTimeOffsets;
_expectedDateTimes = null;
_columnType = columnType;
}

internal SFDataType ExpectedColumnType() => _columnType;

internal void AssertEqual(object actual, string comparisonFormat, string faultMessage)
internal void AssertEqual(object actual, string comparisonFormat, string faultMessage, int index)
{
switch (_columnType)
{
case SFDataType.TIMESTAMP_TZ:
Assert.AreEqual(GetDateTimeOffset().ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimeOffsets()[index].ToString(comparisonFormat), ((DateTimeOffset)actual).ToString(comparisonFormat), faultMessage);
break;
case SFDataType.TIMESTAMP_LTZ:
Assert.AreEqual(GetDateTimeOffset().ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimeOffsets()[index].ToUniversalTime().ToString(comparisonFormat), ((DateTimeOffset)actual).ToUniversalTime().ToString(comparisonFormat), faultMessage);
break;
default:
Assert.AreEqual(GetDateTime().ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
Assert.AreEqual(GetDateTimes()[index].ToString(comparisonFormat), ((DateTime)actual).ToString(comparisonFormat), faultMessage);
break;
}
}

internal DateTime GetDateTime() => _expectedDateTime ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
internal DateTime[] GetDateTimes() => _expectedDateTimes ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");

internal DateTimeOffset GetDateTimeOffset() => _expectedDateTimeOffset ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");
internal DateTimeOffset[] GetDateTimeOffsets() => _expectedDateTimeOffsets ?? throw new Exception($"Column {_columnType} is not matching the expected value type {typeof(DateTime)}");

internal static bool IsOffsetType(SFDataType type) => type == SFDataType.TIMESTAMP_LTZ || type == SFDataType.TIMESTAMP_TZ;
}
Expand Down
19 changes: 14 additions & 5 deletions Snowflake.Data.Tests/UnitTests/SFBindUploaderTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,14 @@ public void TestCsvDataConversionForTime(SFDataType dbType, string input, string
// Assert
Assert.AreEqual(timeExpected, timeActual);
}

[TestCase(SFDataType.TIMESTAMP_LTZ, "39600000000000", "1970-01-01T12:00:00.0000000+01:00")]

[TestCase(SFDataType.TIMESTAMP_LTZ, "0", "1970-01-01T00:00:00.0000000+00:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "39600000000000", "1970-01-01T12:00:00.0000000+01:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "1341136800000000000", "2012-07-01T12:00:00.0000000+02:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "352245599987654000", "1981-02-28T23:59:59.9876540+02:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "1678868249207000000", "2023/03/15T13:17:29.207+05:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "253402300799999999900", "9999-12-31T23:59:59.9999999+00:00")]
[TestCase(SFDataType.TIMESTAMP_LTZ, "-62135596800000000000", "0001-01-01T00:00:00.0000000+00:00")]
public void TestCsvDataConversionForTimestampLtz(SFDataType dbType, string input, string expected)
{
// Arrange
Expand All @@ -60,9 +63,12 @@ public void TestCsvDataConversionForTimestampLtz(SFDataType dbType, string input
// Assert
Assert.AreEqual(timestampExpected.ToLocalTime(), timestampActual);
}


[TestCase(SFDataType.TIMESTAMP_TZ, "0 1440", "1970-01-01 00:00:00.000000 +00:00")]
[TestCase(SFDataType.TIMESTAMP_TZ, "1341136800000000000 1560", "2012-07-01 12:00:00.000000 +02:00")]
[TestCase(SFDataType.TIMESTAMP_TZ, "352245599987654000 1560", "1981-02-28 23:59:59.987654 +02:00")]
[TestCase(SFDataType.TIMESTAMP_TZ, "253402300799999999000 1440", "9999-12-31 23:59:59.999999 +00:00")]
[TestCase(SFDataType.TIMESTAMP_TZ, "-62135596800000000000 1440", "0001-01-01 00:00:00.000000 +00:00")]
public void TestCsvDataConversionForTimestampTz(SFDataType dbType, string input, string expected)
{
// Arrange
Expand All @@ -74,12 +80,15 @@ public void TestCsvDataConversionForTimestampTz(SFDataType dbType, string input,
// Assert
Assert.AreEqual(timestampExpected, timestampActual);
}


[TestCase(SFDataType.TIMESTAMP_NTZ, "0", "1970-01-01 00:00:00.000000")]
[TestCase(SFDataType.TIMESTAMP_NTZ, "1341144000000000000", "2012-07-01 12:00:00.000000")]
[TestCase(SFDataType.TIMESTAMP_NTZ, "352252799987654000", "1981-02-28 23:59:59.987654")]
[TestCase(SFDataType.TIMESTAMP_NTZ, "253402300799999999000", "9999-12-31 23:59:59.999999")]
[TestCase(SFDataType.TIMESTAMP_NTZ, "-62135596800000000000", "0001-01-01 00:00:00.000000")]
public void TestCsvDataConversionForTimestampNtz(SFDataType dbType, string input, string expected)
{
// Arrange
// Arrange
DateTime timestampExpected = DateTime.Parse(expected);
var check = SFDataConverter.csharpValToSfVal(SFDataType.TIMESTAMP_NTZ, timestampExpected);
Assert.AreEqual(check, input);
Expand Down
26 changes: 24 additions & 2 deletions Snowflake.Data.Tests/UnitTests/SFDataConverterTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

using System;
using System.Text;
using Snowflake.Data.Client;
using Snowflake.Data.Tests.Util;

namespace Snowflake.Data.Tests.UnitTests
{
Expand Down Expand Up @@ -36,7 +38,7 @@ public void TestConvertBindToSFValFinlandLocale()

Thread.CurrentThread.CurrentCulture = ci;

System.Tuple<string, string> t =
System.Tuple<string, string> t =
SFDataConverter.csharpTypeValToSfTypeVal(System.Data.DbType.Double, 1.2345);

Assert.AreEqual("REAL", t.Item1);
Expand Down Expand Up @@ -109,7 +111,7 @@ public void TestConvertTimeSpan(string inputTimeStr)
var tickDiff = val.Ticks;
var inputStringAsItComesBackFromDatabase = (tickDiff / 10000000.0m).ToString(CultureInfo.InvariantCulture);
inputStringAsItComesBackFromDatabase += inputTimeStr.Substring(8, inputTimeStr.Length - 8);

// Run the conversion
var result = SFDataConverter.ConvertToCSharpVal(ConvertToUTF8Buffer(inputStringAsItComesBackFromDatabase), SFDataType.TIME, typeof(TimeSpan));

Expand Down Expand Up @@ -326,5 +328,25 @@ public void TestInvalidConversionInvalidDecimal(string s)
Assert.Throws<FormatException>(() => SFDataConverter.ConvertToCSharpVal(ConvertToUTF8Buffer(s), SFDataType.FIXED, typeof(decimal)));
}

[Test]
[TestCase(SFDataType.TIMESTAMP_LTZ, typeof(DateTime))]
[TestCase(SFDataType.TIMESTAMP_TZ, typeof(DateTime))]
[TestCase(SFDataType.TIMESTAMP_NTZ, typeof(DateTimeOffset))]
[TestCase(SFDataType.TIME, typeof(DateTimeOffset))]
[TestCase(SFDataType.DATE, typeof(DateTimeOffset))]
public void TestInvalidTimestampConversion(SFDataType dataType, Type unsupportedType)
{
object unsupportedObject;
if (unsupportedType == typeof(DateTimeOffset))
unsupportedObject = new DateTimeOffset();
else if (unsupportedType == typeof(DateTime))
unsupportedObject = new DateTime();
else
unsupportedObject = null;

Assert.NotNull(unsupportedType);
sfc-gh-knozderko marked this conversation as resolved.
Show resolved Hide resolved
SnowflakeDbException ex = Assert.Throws<SnowflakeDbException>(() => SFDataConverter.csharpValToSfVal(dataType, unsupportedObject));
SnowflakeDbExceptionAssert.HasErrorCode(ex, SFError.INVALID_DATA_CONVERSION);
}
sfc-gh-knozderko marked this conversation as resolved.
Show resolved Hide resolved
}
}
Loading
Loading