diff --git a/runtime/sam/expr/function/fields.go b/runtime/sam/expr/function/fields.go index a9015e442a..d2ae5e06b9 100644 --- a/runtime/sam/expr/function/fields.go +++ b/runtime/sam/expr/function/fields.go @@ -19,8 +19,7 @@ func NewFields(zctx *zed.Context) *Fields { } } -func buildPath(typ *zed.TypeRecord, b *zcode.Builder, prefix []string) []string { - var out []string +func buildPath(typ *zed.TypeRecord, b *zcode.Builder, prefix []string) { for _, f := range typ.Fields { if typ, ok := zed.TypeUnder(f.Type).(*zed.TypeRecord); ok { buildPath(typ, b, append(prefix, f.Name)) @@ -33,17 +32,18 @@ func buildPath(typ *zed.TypeRecord, b *zcode.Builder, prefix []string) []string b.EndContainer() } } - return out } func (f *Fields) Call(ectx expr.Context, args []zed.Value) zed.Value { arena := ectx.Arena() - subjectVal := args[0] + subjectVal := args[0].Under(arena) typ := f.recordType(subjectVal) if typ == nil { return f.zctx.Missing(ectx.Arena()) } - //XXX should have a way to append into allocator + if subjectVal.IsNull() { + return arena.New(f.typ, nil) + } var b zcode.Builder buildPath(typ, &b, nil) return arena.New(f.typ, b.Bytes()) diff --git a/runtime/sam/expr/ztests/fields.yaml b/runtime/sam/expr/ztests/fields.yaml deleted file mode 100644 index 7ee6cfe30d..0000000000 --- a/runtime/sam/expr/ztests/fields.yaml +++ /dev/null @@ -1,9 +0,0 @@ -zed: cut fields(this), fields2:=fields(r) - -input: | - {r:{a:1(int32)},s:123(int32)} - {r:{a:1(int8),b:2(int8)},s:"a"} - -output: | - {fields:[["r","a"],["s"]],fields2:[["a"]]} - {fields:[["r","a"],["r","b"],["s"]],fields2:[["a"],["b"]]} diff --git a/runtime/vam/expr/function/fields.go b/runtime/vam/expr/function/fields.go new file mode 100644 index 0000000000..bdc0bf99f4 --- /dev/null +++ b/runtime/vam/expr/function/fields.go @@ -0,0 +1,94 @@ +package function + +import ( + "github.com/brimdata/zed" + "github.com/brimdata/zed/vector" +) + +// https://github.com/brimdata/zed/blob/main/docs/language/functions.md#fields +type Fields struct { + zctx *zed.Context + innerTyp *zed.TypeArray + outerTyp *zed.TypeArray +} + +func NewFields(zctx *zed.Context) *Fields { + inner := zctx.LookupTypeArray(zed.TypeString) + return &Fields{ + zctx: zctx, + innerTyp: inner, + outerTyp: zctx.LookupTypeArray(inner), + } +} + +func (f *Fields) Call(args ...vector.Any) vector.Any { + val := vector.Under(args[0]) + switch typ := val.Type().(type) { + case *zed.TypeRecord: + paths := buildPath(typ, nil) + s := vector.NewStringEmpty(val.Len(), nil) + inOffs, outOffs := []uint32{0}, []uint32{0} + nulls := vector.NewBoolEmpty(val.Len(), nil) + for i := uint32(0); i < val.Len(); i++ { + if vector.IsNull(val, i) { + nulls.Set(i) + outOffs = append(outOffs, outOffs[len(outOffs)-1]) + continue + } + inOffs, outOffs = appendPaths(paths, s, inOffs, outOffs) + } + inner := vector.NewArray(f.innerTyp, inOffs, s, nil) + return vector.NewArray(f.outerTyp, outOffs, inner, nulls) + case *zed.TypeOfType: + var errcnt uint32 + tags := make([]uint32, val.Len()) + s := vector.NewStringEmpty(val.Len(), nil) + inOffs, outOffs := []uint32{0}, []uint32{0} + for i := uint32(0); i < val.Len(); i++ { + b, _ := vector.TypeValueValue(val, i) + rtyp := f.recordType(b) + if rtyp == nil { + errcnt++ + tags[i] = 1 + continue + } + inOffs, outOffs = appendPaths(buildPath(rtyp, nil), s, inOffs, outOffs) + } + inner := vector.NewArray(f.innerTyp, inOffs, s, nil) + out := vector.NewArray(f.outerTyp, outOffs, inner, nil) + return vector.NewVariant(tags, []vector.Any{out, vector.NewStringError(f.zctx, "missing", errcnt)}) + default: + return vector.NewStringError(f.zctx, "missing", val.Len()) + } +} + +func (f *Fields) recordType(b []byte) *zed.TypeRecord { + typ, err := f.zctx.LookupByValue(b) + if err != nil { + return nil + } + rtyp, _ := typ.(*zed.TypeRecord) + return rtyp +} + +func buildPath(typ *zed.TypeRecord, prefix []string) [][]string { + var out [][]string + for _, f := range typ.Fields { + if typ, ok := zed.TypeUnder(f.Type).(*zed.TypeRecord); ok { + out = append(out, buildPath(typ, append(prefix, f.Name))...) + } else { + out = append(out, append(prefix, f.Name)) + } + } + return out +} + +func appendPaths(paths [][]string, s *vector.String, inner, outer []uint32) ([]uint32, []uint32) { + for _, path := range paths { + for _, f := range path { + s.Append(f) + } + inner = append(inner, inner[len(inner)-1]+uint32(len(path))) + } + return inner, append(outer, outer[len(outer)-1]+uint32(len(paths))) +} diff --git a/runtime/vam/expr/function/function.go b/runtime/vam/expr/function/function.go index d9bac9992b..678c161746 100644 --- a/runtime/vam/expr/function/function.go +++ b/runtime/vam/expr/function/function.go @@ -14,6 +14,8 @@ func New(zctx *zed.Context, name string, narg int) (expr.Function, field.Path, e var path field.Path var f expr.Function switch name { + case "fields": + f = NewFields(zctx) case "join": argmax = 2 f = &Join{zctx: zctx} diff --git a/runtime/ztests/expr/function/fields.yaml b/runtime/ztests/expr/function/fields.yaml new file mode 100644 index 0000000000..d811315bd7 --- /dev/null +++ b/runtime/ztests/expr/function/fields.yaml @@ -0,0 +1,19 @@ +zed: fields(this) + +vector: true + +input: | + {r:{a:1(int32)},s:123(int32)}((string,{r:{a:int32},s:int32})) + null({r:{a:int32},s:int32}) + {r:{a:1(int8),b:2(int8)},s:"a"} + <{r:{a:int32},s:int32}> + <{r:{a:int8,b:int8},s:string}>((int8,type)) + + +output: | + [["r","a"],["s"]] + null([[string]]) + [["r","a"],["r","b"],["s"]] + [["r","a"],["s"]] + [["r","a"],["r","b"],["s"]] + error("missing") diff --git a/vector/bool.go b/vector/bool.go index f112967d19..7a275933d3 100644 --- a/vector/bool.go +++ b/vector/bool.go @@ -70,3 +70,50 @@ func (b *Bool) String() string { } return s.String() } + +func IsNull(v Any, slot uint32) bool { + switch v := Under(v).(type) { + case *View: + return IsNull(v.Any, v.Index[slot]) + case *Const: + return v.val.IsNull() || NullsOf(v).Value(slot) + default: + return NullsOf(v).Value(slot) + } +} + +func NullsOf(v Any) *Bool { + switch v := v.(type) { + case *Array: + return v.Nulls + case *Bytes: + return v.Nulls + case *Const: + return v.Nulls + case *Dict: + return v.Nulls + case *Error: + return v.Nulls + case *Float: + return v.Nulls + case *Int: + return v.Nulls + case *Map: + return v.Nulls + case *Net: + return v.Nulls + case *Record: + return v.Nulls + case *Set: + return v.Nulls + case *String: + return v.Nulls + case *TypeValue: + return v.Nulls + case *Uint: + return v.Nulls + case *Union: + return v.Nulls + } + panic(v) +}