Move datatype folders to super folder (#1583)

Moves geo, hstore, interval and numeric files where we do special operations on those types, to a `datatypes` folder
PeerDB-io · Apr 8, 2024 · d8a10c3 · d8a10c3
1 parent 55d5b3f
commit d8a10c3
Show file tree

Hide file tree

Showing 19 changed files with 28 additions and 29 deletions.
diff --git a/flow/connectors/bigquery/bigquery.go b/flow/connectors/bigquery/bigquery.go
@@ -19,10 +19,10 @@ import (
 
 	metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata"
 	"github.com/PeerDB-io/peer-flow/connectors/utils"
+	numeric "github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
 	"github.com/PeerDB-io/peer-flow/logger"
 	"github.com/PeerDB-io/peer-flow/model"
-	"github.com/PeerDB-io/peer-flow/model/numeric"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
 	"github.com/PeerDB-io/peer-flow/peerdbenv"
 	"github.com/PeerDB-io/peer-flow/shared"

diff --git a/flow/connectors/bigquery/qrep_avro_sync.go b/flow/connectors/bigquery/qrep_avro_sync.go
@@ -14,9 +14,9 @@ import (
 	"go.temporal.io/sdk/activity"
 
 	avro "github.com/PeerDB-io/peer-flow/connectors/utils/avro"
+	numeric "github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
 	"github.com/PeerDB-io/peer-flow/model"
-	"github.com/PeerDB-io/peer-flow/model/numeric"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
 	"github.com/PeerDB-io/peer-flow/shared"
 )

diff --git a/flow/connectors/clickhouse/normalize.go b/flow/connectors/clickhouse/normalize.go
@@ -8,9 +8,9 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
 	"github.com/PeerDB-io/peer-flow/model"
-	"github.com/PeerDB-io/peer-flow/model/numeric"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
 )
 
@@ -84,10 +84,10 @@ func generateCreateTableSQLForNormalizedTable(
 
 		switch colType {
 		case qvalue.QValueKindNumeric:
-			precision, scale := numeric.ParseNumericTypmod(column.TypeModifier)
+			precision, scale := datatypes.ParseNumericTypmod(column.TypeModifier)
 			if column.TypeModifier == -1 || precision > 76 || scale > precision {
-				precision = numeric.PeerDBClickhousePrecision
-				scale = numeric.PeerDBClickhouseScale
+				precision = datatypes.PeerDBClickhousePrecision
+				scale = datatypes.PeerDBClickhouseScale
 			}
 			stmtBuilder.WriteString(fmt.Sprintf("`%s` DECIMAL(%d, %d), ",
 				colName, precision, scale))

diff --git a/flow/connectors/postgres/cdc.go b/flow/connectors/postgres/cdc.go
@@ -19,8 +19,8 @@ import (
 
 	"github.com/PeerDB-io/peer-flow/connectors/utils"
 	"github.com/PeerDB-io/peer-flow/connectors/utils/cdc_records"
+	geo "github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
-	"github.com/PeerDB-io/peer-flow/geo"
 	"github.com/PeerDB-io/peer-flow/logger"
 	"github.com/PeerDB-io/peer-flow/model"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"

diff --git a/flow/connectors/postgres/client.go b/flow/connectors/postgres/client.go
@@ -13,9 +13,9 @@ import (
 	"github.com/lib/pq/oid"
 
 	"github.com/PeerDB-io/peer-flow/connectors/utils"
+	numeric "github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
 	"github.com/PeerDB-io/peer-flow/model"
-	"github.com/PeerDB-io/peer-flow/model/numeric"
 	"github.com/PeerDB-io/peer-flow/shared"
 )
 

diff --git a/flow/connectors/postgres/qrep_query_executor.go b/flow/connectors/postgres/qrep_query_executor.go
@@ -10,9 +10,8 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 	"go.temporal.io/sdk/log"
 
-	"github.com/PeerDB-io/peer-flow/geo"
+	datatypes "github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/model"
-	"github.com/PeerDB-io/peer-flow/model/numeric"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
 	"github.com/PeerDB-io/peer-flow/shared"
 )
@@ -79,7 +78,7 @@ func (qe *QRepQueryExecutor) fieldDescriptionsToSchema(fds []pgconn.FieldDescrip
 		// TODO fix this.
 		cnullable := true
 		if ctype == qvalue.QValueKindNumeric {
-			precision, scale := numeric.ParseNumericTypmod(fd.TypeModifier)
+			precision, scale := datatypes.ParseNumericTypmod(fd.TypeModifier)
 			qfields[i] = qvalue.QField{
 				Name:      cname,
 				Type:      ctype,
@@ -426,7 +425,7 @@ func (qe *QRepQueryExecutor) mapRowToQRecord(
 				switch customQKind {
 				case qvalue.QValueKindGeography, qvalue.QValueKindGeometry:
 					wkbString, ok := values[i].(string)
-					wkt, err := geo.GeoValidate(wkbString)
+					wkt, err := datatypes.GeoValidate(wkbString)
 					if err != nil || !ok {
 						record[i] = qvalue.QValueNull(qvalue.QValueKindGeography)
 					} else if customQKind == qvalue.QValueKindGeography {

diff --git a/flow/connectors/postgres/qvalue_convert.go b/flow/connectors/postgres/qvalue_convert.go
@@ -13,7 +13,7 @@ import (
 	"github.com/lib/pq/oid"
 	"github.com/shopspring/decimal"
 
-	peerdb_interval "github.com/PeerDB-io/peer-flow/interval"
+	datatypes "github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
 	"github.com/PeerDB-io/peer-flow/shared"
 )
@@ -229,7 +229,7 @@ func parseFieldFromQValueKind(qvalueKind qvalue.QValueKind, value interface{}) (
 		return qvalue.QValueTimestampTZ{Val: timestamp}, nil
 	case qvalue.QValueKindInterval:
 		intervalObject := value.(pgtype.Interval)
-		var interval peerdb_interval.PeerDBInterval
+		var interval datatypes.PeerDBInterval
 		interval.Hours = int(intervalObject.Microseconds / 3600000000)
 		interval.Minutes = int((intervalObject.Microseconds % 3600000000) / 60000000)
 		interval.Seconds = float64(intervalObject.Microseconds%60000000) / 1000000.0

diff --git a/flow/connectors/snowflake/merge_stmt_generator.go b/flow/connectors/snowflake/merge_stmt_generator.go
@@ -5,8 +5,8 @@ import (
 	"strings"
 
 	"github.com/PeerDB-io/peer-flow/connectors/utils"
+	numeric "github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
-	"github.com/PeerDB-io/peer-flow/model/numeric"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
 	"github.com/PeerDB-io/peer-flow/shared"
 )

diff --git a/flow/connectors/snowflake/snowflake.go b/flow/connectors/snowflake/snowflake.go
@@ -21,10 +21,10 @@ import (
 
 	metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata"
 	"github.com/PeerDB-io/peer-flow/connectors/utils"
+	numeric "github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
 	"github.com/PeerDB-io/peer-flow/logger"
 	"github.com/PeerDB-io/peer-flow/model"
-	"github.com/PeerDB-io/peer-flow/model/numeric"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
 	"github.com/PeerDB-io/peer-flow/peerdbenv"
 	"github.com/PeerDB-io/peer-flow/shared"

diff --git a/flow/geo/geo.go → flow/datatypes/geo.go b/flow/geo/geo.go → flow/datatypes/geo.go
@@ -1,4 +1,4 @@
-package geo
+package datatypes
 
 import (
 	"encoding/hex"

diff --git a/flow/hstore/hstore.go → flow/datatypes/hstore.go b/flow/hstore/hstore.go → flow/datatypes/hstore.go
@@ -5,7 +5,7 @@ https://github.com/postgres/postgres/blob/bea18b1c949145ba2ca79d4765dba3cc9494a4
 This package is an implementation based on the above code.
 It's simplified to only parse the subset which `hstore_out` outputs.
 */
-package hstore_util
+package datatypes
 
 import (
 	"encoding/json"

diff --git a/flow/hstore/hstore_test.go → flow/datatypes/hstore_test.go b/flow/hstore/hstore_test.go → flow/datatypes/hstore_test.go
@@ -1,4 +1,4 @@
-package hstore_util
+package datatypes
 
 import (
 	"testing"

diff --git a/flow/interval/interval.go → flow/datatypes/interval.go b/flow/interval/interval.go → flow/datatypes/interval.go
@@ -1,4 +1,4 @@
-package peerdb_interval
+package datatypes
 
 type PeerDBInterval struct {
 	Hours   int     `json:"hours,omitempty"`

diff --git a/flow/model/numeric/scale.go → flow/datatypes/numeric.go b/flow/model/numeric/scale.go → flow/datatypes/numeric.go
@@ -1,4 +1,4 @@
-package numeric
+package datatypes
 
 const (
 	PeerDBNumericPrecision    = 38

diff --git a/flow/model/qrecord_batch.go b/flow/model/qrecord_batch.go
@@ -10,7 +10,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5/pgtype"
 
-	"github.com/PeerDB-io/peer-flow/geo"
+	geo "github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
 )
 

diff --git a/flow/model/qvalue/avro_converter.go b/flow/model/qvalue/avro_converter.go
@@ -13,8 +13,8 @@ import (
 	"github.com/shopspring/decimal"
 	"go.temporal.io/sdk/log"
 
+	"github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
-	hstore_util "github.com/PeerDB-io/peer-flow/hstore"
 )
 
 // https://avro.apache.org/docs/1.11.0/spec.html
@@ -544,7 +544,7 @@ func (c *QValueAvroConverter) processArrayDate(arrayDate []time.Time) interface{
 }
 
 func (c *QValueAvroConverter) processHStore(hstore string) (interface{}, error) {
-	jsonString, err := hstore_util.ParseHstore(hstore)
+	jsonString, err := datatypes.ParseHstore(hstore)
 	if err != nil {
 		return "", fmt.Errorf("cannot parse %s: %w", hstore, err)
 	}

diff --git a/flow/model/qvalue/dwh.go b/flow/model/qvalue/dwh.go
@@ -5,8 +5,8 @@ import (
 
 	"go.temporal.io/sdk/log"
 
+	numeric "github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/generated/protos"
-	"github.com/PeerDB-io/peer-flow/model/numeric"
 )
 
 func DetermineNumericSettingForDWH(precision int16, scale int16, dwh protos.DBType) (int16, int16) {

diff --git a/flow/model/qvalue/equals.go b/flow/model/qvalue/equals.go
@@ -14,7 +14,7 @@ import (
 	"github.com/shopspring/decimal"
 	geom "github.com/twpayne/go-geos"
 
-	hstore_util "github.com/PeerDB-io/peer-flow/hstore"
+	"github.com/PeerDB-io/peer-flow/datatypes"
 )
 
 func valueEmpty(value any) bool {
@@ -174,7 +174,7 @@ func compareHStore(str1 string, value2 interface{}) bool {
 	if str1 == str2 {
 		return true
 	}
-	parsedHStore1, err := hstore_util.ParseHstore(str1)
+	parsedHStore1, err := datatypes.ParseHstore(str1)
 	if err != nil {
 		panic(err)
 	}

diff --git a/flow/model/record_items.go b/flow/model/record_items.go
@@ -7,7 +7,7 @@ import (
 
 	"github.com/google/uuid"
 
-	hstore_util "github.com/PeerDB-io/peer-flow/hstore"
+	"github.com/PeerDB-io/peer-flow/datatypes"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
 )
 
@@ -132,7 +132,7 @@ func (r RecordItems) toMap(hstoreAsJSON bool, opts ToJSONOptions) (map[string]in
 			if !hstoreAsJSON {
 				jsonStruct[col] = hstoreVal
 			} else {
-				jsonVal, err := hstore_util.ParseHstore(hstoreVal)
+				jsonVal, err := datatypes.ParseHstore(hstoreVal)
 				if err != nil {
 					return nil, fmt.Errorf("unable to convert hstore column %s to json for value %T: %w", col, v, err)
 				}