From d2832f17effa1d32a1337908d13e2ce0bcae8128 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Thu, 4 Jan 2024 18:08:48 +0530 Subject: [PATCH 1/3] cast in qrep --- flow/connectors/bigquery/qrep_avro_sync.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flow/connectors/bigquery/qrep_avro_sync.go b/flow/connectors/bigquery/qrep_avro_sync.go index a3f9e73aa7..7f7b1d5b83 100644 --- a/flow/connectors/bigquery/qrep_avro_sync.go +++ b/flow/connectors/bigquery/qrep_avro_sync.go @@ -123,6 +123,9 @@ func getTransformedColumns(dstTableMetadata *bigquery.TableMetadata, syncedAtCol case bigquery.JSONFieldType: transformedColumns = append(transformedColumns, fmt.Sprintf("PARSE_JSON(`%s`) AS `%s`", col.Name, col.Name)) + case bigquery.DateFieldType: + transformedColumns = append(transformedColumns, + fmt.Sprintf("CAST(`%s` AS DATE) AS `%s`", col.Name, col.Name)) default: transformedColumns = append(transformedColumns, fmt.Sprintf("`%s`", col.Name)) } From 909a2c7ac5287c3ede90c57c14a78c465b3f21a0 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Thu, 4 Jan 2024 18:20:55 +0530 Subject: [PATCH 2/3] adds test for date --- flow/e2e/test_utils.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flow/e2e/test_utils.go b/flow/e2e/test_utils.go index 99903fbdfa..ed9a606ab2 100644 --- a/flow/e2e/test_utils.go +++ b/flow/e2e/test_utils.go @@ -222,6 +222,7 @@ func CreateTableForQRep(pool *pgxpool.Pool, suffix string, tableName string) err "f6 jsonb", "f7 jsonb", "f8 smallint", + "my_date DATE", } if strings.Contains(tableName, "sf") || strings.Contains(tableName, "bq") { tblFields = append(tblFields, `"geometryPoint" geometry(point)`, @@ -282,7 +283,7 @@ func PopulateSourceTable(pool *pgxpool.Pool, suffix string, tableName string, ro CURRENT_TIMESTAMP, 1, ARRAY['text1', 'text2'], ARRAY[123, 456], ARRAY[789, 012], ARRAY['varchar1', 'varchar2'], '{"key": 8.5}', '[{"key1": "value1", "key2": "value2", "key3": "value3"}]', - '{"key": "value"}', 15 %s + '{"key": "value"}', 15, '2023-08-08' %s )`, id, uuid.New().String(), uuid.New().String(), uuid.New().String(), uuid.New().String(), uuid.New().String(), uuid.New().String(), geoValues) @@ -303,7 +304,7 @@ func PopulateSourceTable(pool *pgxpool.Pool, suffix string, tableName string, ro deal_id, ethereum_transaction_id, ignore_price, card_eth_value, paid_eth_price, card_bought_notified, address, account_id, asset_id, status, transaction_id, settled_at, reference_id, - settle_at, settlement_delay_reason, f1, f2, f3, f4, f5, f6, f7, f8 + settle_at, settlement_delay_reason, f1, f2, f3, f4, f5, f6, f7, f8, my_date %s ) VALUES %s; `, suffix, tableName, geoColumns, strings.Join(rows, ","))) @@ -424,6 +425,7 @@ func GetOwnersSchema() *model.QRecordSchema { {Name: "f6", Type: qvalue.QValueKindJSON, Nullable: true}, {Name: "f7", Type: qvalue.QValueKindJSON, Nullable: true}, {Name: "f8", Type: qvalue.QValueKindInt16, Nullable: true}, + {Name: "my_date", Type: qvalue.QValueKindDate, Nullable: true}, {Name: "geometryPoint", Type: qvalue.QValueKindGeometry, Nullable: true}, {Name: "geometry_linestring", Type: qvalue.QValueKindGeometry, Nullable: true}, {Name: "geometry_polygon", Type: qvalue.QValueKindGeometry, Nullable: true}, From 89c9ac14c2491b815c1a4576df830b7d4e8f0f07 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Thu, 4 Jan 2024 18:23:25 +0530 Subject: [PATCH 3/3] use current_date instead --- flow/e2e/test_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/e2e/test_utils.go b/flow/e2e/test_utils.go index ed9a606ab2..a669691090 100644 --- a/flow/e2e/test_utils.go +++ b/flow/e2e/test_utils.go @@ -283,7 +283,7 @@ func PopulateSourceTable(pool *pgxpool.Pool, suffix string, tableName string, ro CURRENT_TIMESTAMP, 1, ARRAY['text1', 'text2'], ARRAY[123, 456], ARRAY[789, 012], ARRAY['varchar1', 'varchar2'], '{"key": 8.5}', '[{"key1": "value1", "key2": "value2", "key3": "value3"}]', - '{"key": "value"}', 15, '2023-08-08' %s + '{"key": "value"}', 15, CURRENT_DATE %s )`, id, uuid.New().String(), uuid.New().String(), uuid.New().String(), uuid.New().String(), uuid.New().String(), uuid.New().String(), geoValues)