handle float32 separately

flow/model/model.go

@@ -15,6 +15,7 @@ import (
 	"github.com/PeerDB-io/peer-flow/generated/protos"
 	"github.com/PeerDB-io/peer-flow/model/qvalue"
 	"github.com/PeerDB-io/peer-flow/peerdbenv"
+	"golang.org/x/exp/constraints"
 )
 
 type NameAndExclude struct {
@@ -132,10 +133,10 @@ func (r *RecordItems) Len() int {
 	return len(r.Values)
 }
 
-func convertFloatArrayToStringArray(floatArr []float64) []string {
+func convertFloatArrayToStringArray[T constraints.Float](floatArr []T) []string {
 	strArr := make([]string, len(floatArr))
 	for i, v := range floatArr {
-		strArr[i] = strconv.FormatFloat(v, 'f', -1, 64) // Convert float to string
+		strArr[i] = strconv.FormatFloat(float64(v), 'f', -1, 64)
 	}
 	return strArr
 }
@@ -178,7 +179,7 @@ func (r *RecordItems) toMap() (map[string]interface{}, error) {
 				return nil, errors.New("expected *big.Rat value")
 			}
 			jsonStruct[col] = bigRat.FloatString(9)
-		case qvalue.QValueKindFloat64, qvalue.QValueKindFloat32:
+		case qvalue.QValueKindFloat64:
 			floatVal, ok := v.Value.(float64)
 			if !ok {
 				return nil, errors.New("expected float64 value")
@@ -188,7 +189,17 @@ func (r *RecordItems) toMap() (map[string]interface{}, error) {
 			} else {
 				jsonStruct[col] = floatVal
 			}
-		case qvalue.QValueKindArrayFloat64, qvalue.QValueKindArrayFloat32:
+		case qvalue.QValueKindFloat32:
+			floatVal, ok := v.Value.(float32)
+			if !ok {
+				return nil, errors.New("expected float64 value")
+			}
+			if math.IsNaN(float64(floatVal)) {
+				jsonStruct[col] = nil
+			} else {
+				jsonStruct[col] = floatVal
+			}
+		case qvalue.QValueKindArrayFloat64:
 			floatArr, ok := v.Value.([]float64)
 			if !ok {
 				return nil, errors.New("expected []float64 value")
@@ -207,6 +218,25 @@ func (r *RecordItems) toMap() (map[string]interface{}, error) {
 				}
 			}
 			jsonStruct[col] = cleanedArr
+		case qvalue.QValueKindArrayFloat32:
+			floatArr, ok := v.Value.([]float32)
+			if !ok {
+				return nil, errors.New("expected []float32 value")
+			}
+
+			// json.Marshal cannot support NaN or INF values
+			// BigQuery cannot support NULL values in arrays
+			// Solution: Skip NaN values in float arrays we get
+			cleanedArr := make([]float32, 0, len(floatArr))
+			for _, val := range floatArr {
+				if !math.IsNaN(float64(val)) && !math.IsInf(float64(val), 0) {
+					cleanedArr = append(cleanedArr, val)
+				} else {
+					slog.Warn("NaN or INF value found in float array - skipping",
+						slog.Any("array", convertFloatArrayToStringArray(floatArr)))
+				}
+			}
+			jsonStruct[col] = cleanedArr
 		default:
 			jsonStruct[col] = v.Value
 		}