Skip to content

Commit

Permalink
fix: ignore nil value in array convert (#2748)
Browse files Browse the repository at this point in the history 
Signed-off-by: carlclone <[email protected]>
  • Loading branch information
@carlclone @ngjaying
carlclone authored and ngjaying committed Apr 22, 2024
1 parent 7fd9a87 commit 4fb2095
Show file tree
Hide file tree
Showing 6 changed files with 78 additions and 20 deletions.
12 changes: 6 additions & 6 deletions internal/binder/function/funcs_agg.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ func registerAggFunc() {
exec: func(ctx api.FunctionContext, args []interface{}) (interface{}, bool) {
arg0 := args[0].([]interface{})
if len(arg0) > 0 {
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND)
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND, cast.IGNORE_NIL)
if err != nil {
return fmt.Errorf("requires float64 slice but found %[1]T(%[1]v)", arg0), false
}
Expand All @@ -203,7 +203,7 @@ func registerAggFunc() {
exec: func(ctx api.FunctionContext, args []interface{}) (interface{}, bool) {
arg0 := args[0].([]interface{})
if len(arg0) > 0 {
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND)
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND, cast.IGNORE_NIL)
if err != nil {
return fmt.Errorf("requires float64 slice but found %[1]T(%[1]v)", arg0), false
}
Expand All @@ -226,7 +226,7 @@ func registerAggFunc() {
exec: func(ctx api.FunctionContext, args []interface{}) (interface{}, bool) {
arg0 := args[0].([]interface{})
if len(arg0) > 0 {
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND)
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND, cast.IGNORE_NIL)
if err != nil {
return fmt.Errorf("requires float64 slice but found %[1]T(%[1]v)", arg0), false
}
Expand All @@ -249,7 +249,7 @@ func registerAggFunc() {
exec: func(ctx api.FunctionContext, args []interface{}) (interface{}, bool) {
arg0 := args[0].([]interface{})
if len(arg0) > 0 {
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND)
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND, cast.IGNORE_NIL)
if err != nil {
return fmt.Errorf("requires float64 slice but found %[1]T(%[1]v)", arg0), false
}
Expand Down Expand Up @@ -286,7 +286,7 @@ func registerAggFunc() {
}

if len(arg0) > 0 {
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND)
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND, cast.IGNORE_NIL)
if err != nil {
return fmt.Errorf("requires float64 slice but found %[1]T(%[1]v)", arg0), false
}
Expand Down Expand Up @@ -322,7 +322,7 @@ func registerAggFunc() {
arg1Float64 = val
}
if len(arg0) > 0 {
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND)
float64Slice, err := cast.ToFloat64Slice(arg0, cast.CONVERT_SAMEKIND, cast.IGNORE_NIL)
if err != nil {
return fmt.Errorf("requires float64 slice but found %[1]T(%[1]v)", arg0), false
}
Expand Down
51 changes: 42 additions & 9 deletions internal/binder/function/funcs_agg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,8 @@ func TestAggExec(t *testing.T) {
stddevs: fmt.Errorf("requires float64 slice but found []interface {}([foo bar self])"),
var1: fmt.Errorf("requires float64 slice but found []interface {}([foo bar self])"),
vars: fmt.Errorf("requires float64 slice but found []interface {}([foo bar self])"),
}, { // 1
},
{ // 1
args: []interface{}{
[]interface{}{
int64(100),
Expand All @@ -102,7 +103,8 @@ func TestAggExec(t *testing.T) {
stddevs: float64(50),
var1: 1666.6666666666667,
vars: float64(2500),
}, { // 2
},
{ // 2
args: []interface{}{
[]interface{}{
float64(100),
Expand All @@ -117,7 +119,8 @@ func TestAggExec(t *testing.T) {
stddevs: float64(50),
var1: 1666.6666666666667,
vars: float64(2500),
}, { // 3
},
{ // 3
args: []interface{}{
[]interface{}{
100, 150, 200,
Expand All @@ -130,7 +133,8 @@ func TestAggExec(t *testing.T) {
stddevs: float64(50),
var1: 1666.6666666666667,
vars: float64(2500),
}, { // 4
},
{ // 4
args: []interface{}{
[]interface{}{},
},
Expand All @@ -142,6 +146,20 @@ func TestAggExec(t *testing.T) {
var1: nil,
vars: nil,
},
{ // 5
args: []interface{}{
[]interface{}{
100, 150, nil, 200,
},
},
avg: int64(150),
max: int64(200),
min: int64(100),
stddev: 40.824829046386306,
stddevs: float64(50),
var1: 1666.6666666666667,
vars: float64(2500),
},
}
for i, tt := range tests {
rAvg, _ := fAvg.exec(fctx, tt.args)
Expand Down Expand Up @@ -204,7 +222,8 @@ func TestPercentileExec(t *testing.T) {
},
pCont: fmt.Errorf("requires float64 slice but found []interface {}([foo bar self])"),
pDisc: fmt.Errorf("requires float64 slice but found []interface {}([foo bar self])"),
}, { // 1
},
{ // 1
args: []interface{}{
[]interface{}{
int64(100),
Expand All @@ -214,7 +233,8 @@ func TestPercentileExec(t *testing.T) {
},
pCont: fmt.Errorf("Expect 2 arguments but found 1."),
pDisc: fmt.Errorf("Expect 2 arguments but found 1."),
}, { // 2
},
{ // 2
args: []interface{}{
[]interface{}{
int64(100),
Expand All @@ -225,7 +245,8 @@ func TestPercentileExec(t *testing.T) {
},
pCont: float64(125),
pDisc: float64(150),
}, { // 3
},
{ // 3
args: []interface{}{
[]interface{}{
float64(100),
Expand All @@ -236,7 +257,8 @@ func TestPercentileExec(t *testing.T) {
},
pCont: float64(125),
pDisc: float64(150),
}, { // 4
},
{ // 4
args: []interface{}{
[]interface{}{
100, 150, 200,
Expand All @@ -245,14 +267,25 @@ func TestPercentileExec(t *testing.T) {
},
pCont: float64(125),
pDisc: float64(150),
}, { // 5
},
{ // 5
args: []interface{}{
[]interface{}{},
[]interface{}{},
},
pCont: nil,
pDisc: nil,
},
{ // 6
args: []interface{}{
[]interface{}{
100, 150, nil, 200,
},
[]interface{}{0.5, 0.5, 0.5},
},
pCont: float64(125),
pDisc: float64(150),
},
}
for i, tt := range tests {
rCont, _ := pCont.exec(fctx, tt.args)
Expand Down
4 changes: 2 additions & 2 deletions internal/converter/protobuf/fieldConverterSingleton.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ func (fc *FieldConverter) EncodeField(field *desc.FieldDescriptor, v interface{}
)
switch ft {
case dpb.FieldDescriptorProto_TYPE_DOUBLE:
result, err = cast.ToFloat64Slice(v, cast.CONVERT_SAMEKIND)
result, err = cast.ToFloat64Slice(v, cast.CONVERT_SAMEKIND, cast.FORCE_CONVERT)
case dpb.FieldDescriptorProto_TYPE_FLOAT:
result, err = cast.ToTypedSlice(v, func(input interface{}, sn cast.Strictness) (interface{}, error) {
r, err := cast.ToFloat32(input, sn)
Expand Down Expand Up @@ -241,7 +241,7 @@ func (fc *FieldConverter) DecodeField(src interface{}, field *desc.FieldDescript
switch field.GetType() {
case dpb.FieldDescriptorProto_TYPE_DOUBLE, dpb.FieldDescriptorProto_TYPE_FLOAT:
if field.IsRepeated() {
r, e = cast.ToFloat64Slice(src, sn)
r, e = cast.ToFloat64Slice(src, sn, cast.FORCE_CONVERT)
} else {
r, e = cast.ToFloat64(src, sn)
}
Expand Down
2 changes: 1 addition & 1 deletion internal/io/edgex/edgex_sink.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,7 @@ func getValueByType(v interface{}, vt string) (interface{}, error) {
return cast.ToFloat32(input, sn)
}, "float32", cast.CONVERT_SAMEKIND)
case v3.ValueTypeFloat64Array:
return cast.ToFloat64Slice(v, cast.CONVERT_SAMEKIND)
return cast.ToFloat64Slice(v, cast.CONVERT_SAMEKIND, cast.FORCE_CONVERT)
case v3.ValueTypeStringArray:
return cast.ToStringSlice(v, cast.CONVERT_SAMEKIND)
case v3.ValueTypeBinary:
Expand Down
25 changes: 23 additions & 2 deletions pkg/cast/cast.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,13 @@ const (
CONVERT_ALL
)

type ArrayNilConvert int8

const (
IGNORE_NIL ArrayNilConvert = iota
FORCE_CONVERT
)

/*********** Type Cast Utilities *****/

// TODO datetime type
Expand Down Expand Up @@ -411,7 +418,12 @@ func ToFloat64(input interface{}, sn Strictness) (float64, error) {
}
return 0, nil
}
case nil:
if sn == CONVERT_ALL {
return 0, nil
}
}

return 0, fmt.Errorf("cannot convert %[1]T(%[1]v) to float64", input)
}

Expand Down Expand Up @@ -475,6 +487,10 @@ func ToFloat32(input interface{}, sn Strictness) (float32, error) {
}
return 0, nil
}
case nil:
if sn == CONVERT_ALL {
return 0, nil
}
}
return 0, fmt.Errorf("cannot convert %[1]T(%[1]v) to float64", input)
}
Expand Down Expand Up @@ -905,14 +921,19 @@ func ToUint64Slice(input interface{}, sn Strictness) ([]uint64, error) {
return result, nil
}

func ToFloat64Slice(input interface{}, sn Strictness) ([]float64, error) {
func ToFloat64Slice(input interface{}, sn Strictness, anc ArrayNilConvert) ([]float64, error) {
s := reflect.ValueOf(input)
if s.Kind() != reflect.Slice {
return nil, fmt.Errorf("cannot convert %[1]T(%[1]v) to float slice)", input)
}
var result []float64
for i := 0; i < s.Len(); i++ {
ele, err := ToFloat64(s.Index(i).Interface(), sn)
v := s.Index(i).Interface()
if anc == IGNORE_NIL && v == nil {
continue
}

ele, err := ToFloat64(v, sn)
if err != nil {
return nil, fmt.Errorf("cannot convert %[1]T(%[1]v) to float slice for the %d element: %v", input, i, err)
}
Expand Down
4 changes: 4 additions & 0 deletions pkg/cast/cast_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,10 @@ func TestToFloatResult(t *testing.T) {
true,
1,
},
{
nil,
0,
},
}
for _, tt := range tests {
var (
Expand Down

0 comments on commit 4fb2095

Please sign in to comment.