Skip to content

Commit

Permalink
Numeric: adjust precision handling (#1130)
Browse files Browse the repository at this point in the history
`200` was being synced as `199.99999999` in initial load for PostgreSQL
-> Snowflake. It resided in a numeric column.

Noticed that in ToJSON where we transform numerics and other types for
raw table insertion, we set the scale to 9 to conform with the scale
value we've chosen for Snowflake for Numeric values (Number(38,9)). And
this issue isn't present in CDC.

This PR does the same at `processNumeric` in the Avro transform step
where we write to Avro files
  • Loading branch information
Amogh-Bharadwaj authored Jan 23, 2024
1 parent 7042307 commit b87b251
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 1 deletion.
3 changes: 2 additions & 1 deletion flow/model/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/PeerDB-io/peer-flow/generated/protos"
hstore_util "github.com/PeerDB-io/peer-flow/hstore"
"github.com/PeerDB-io/peer-flow/model/numeric"
"github.com/PeerDB-io/peer-flow/model/qvalue"
"github.com/PeerDB-io/peer-flow/peerdbenv"
)
Expand Down Expand Up @@ -215,7 +216,7 @@ func (r *RecordItems) toMap(hstoreAsJSON bool) (map[string]interface{}, error) {
if !ok {
return nil, errors.New("expected *big.Rat value")
}
jsonStruct[col] = bigRat.FloatString(9)
jsonStruct[col] = bigRat.FloatString(numeric.PeerDBNumericScale)
case qvalue.QValueKindFloat64:
floatVal, ok := v.Value.(float64)
if !ok {
Expand Down
3 changes: 3 additions & 0 deletions flow/model/numeric/scale.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
package numeric

const PeerDBNumericScale = 9
4 changes: 4 additions & 0 deletions flow/model/qvalue/avro_converter.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"time"

hstore_util "github.com/PeerDB-io/peer-flow/hstore"
"github.com/PeerDB-io/peer-flow/model/numeric"
"github.com/google/uuid"
"github.com/linkedin/goavro/v2"
)
Expand Down Expand Up @@ -323,6 +324,9 @@ func (c *QValueAvroConverter) processNumeric() (interface{}, error) {
return nil, fmt.Errorf("invalid Numeric value: expected *big.Rat, got %T", c.Value.Value)
}

scale := numeric.PeerDBNumericScale
decimalValue := num.FloatString(scale)
num.SetString(decimalValue)
if c.Nullable {
return goavro.Union("bytes.decimal", num), nil
}
Expand Down

0 comments on commit b87b251

Please sign in to comment.