clickhouse: convert uuid to uuid, not string (#1284)

PeerDB-io · Feb 13, 2024 · c47dbb2 · c47dbb2
1 parent a6c5033
commit c47dbb2
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 5 deletions.
diff --git a/flow/connectors/clickhouse/normalize.go b/flow/connectors/clickhouse/normalize.go
@@ -164,7 +164,7 @@ func (c *ClickhouseConnector) NormalizeRecords(ctx context.Context, req *model.N
 			if err != nil {
 				return nil, fmt.Errorf("error while converting column type to clickhouse type: %w", err)
 			}
-			if clickhouseType == "DateTime64(6)" {
+			if clickhouseType == "DateTime64(6)" || clickhouseType == "UUID" {
 				clickhouseType = "String"
 			}
 

diff --git a/flow/connectors/clickhouse/qvalue_convert.go b/flow/connectors/clickhouse/qvalue_convert.go
@@ -23,6 +23,7 @@ var clickhouseTypeToQValueKindMap = map[string]qvalue.QValueKind{
 	"TIMESTAMP_NTZ":  qvalue.QValueKindTimestamp,
 	"TIMESTAMP_TZ":   qvalue.QValueKindTimestampTZ,
 	"TIME":           qvalue.QValueKindTime,
+	"UUID":           qvalue.QValueKindUUID,
 	"DATE":           qvalue.QValueKindDate,
 	"BLOB":           qvalue.QValueKindBytes,
 	"BYTEA":          qvalue.QValueKindBytes,

diff --git a/flow/model/qvalue/avro_converter.go b/flow/model/qvalue/avro_converter.go
@@ -37,6 +37,11 @@ type AvroSchemaRecord struct {
 	Fields []AvroSchemaField `json:"fields"`
 }
 
+type AvroSchemaLogical struct {
+	Type        string `json:"type"`
+	LogicalType string `json:"logicalType,omitempty"`
+}
+
 type AvroSchemaField struct {
 	Name        string      `json:"name"`
 	Type        interface{} `json:"type"`
@@ -59,8 +64,13 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH QDWHType, precision
 	}
 
 	switch kind {
-	case QValueKindString, QValueKindUUID:
+	case QValueKindString:
 		return "string", nil
+	case QValueKindUUID:
+		return AvroSchemaLogical{
+			Type:        "string",
+			LogicalType: "uuid",
+		}, nil
 	case QValueKindGeometry, QValueKindGeography, QValueKindPoint:
 		return "string", nil
 	case QValueKindInt16, QValueKindInt32, QValueKindInt64:
@@ -86,8 +96,7 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH QDWHType, precision
 				return "string", nil
 			}
 			if kind == QValueKindDate {
-				return AvroSchemaField{
-					Name:        "date",
+				return AvroSchemaLogical{
 					Type:        "int",
 					LogicalType: "date",
 				}, nil

diff --git a/flow/model/qvalue/kind.go b/flow/model/qvalue/kind.go
@@ -110,7 +110,7 @@ var QValueKindToClickhouseTypeMap = map[QValueKind]string{
 	QValueKindBit:         "Boolean",
 	QValueKindBytes:       "String",
 	QValueKindStruct:      "String",
-	QValueKindUUID:        "String",
+	QValueKindUUID:        "UUID",
 	QValueKindTimeTZ:      "String",
 	QValueKindInvalid:     "String",
 	QValueKindHStore:      "String",