From ea8314664645d2a53138db15bc6e7a5869b02f12 Mon Sep 17 00:00:00 2001 From: Matthew Nibecker Date: Wed, 11 Oct 2023 10:24:16 -0700 Subject: [PATCH] Dynamic fields in rename Add the ability to rename fields dynamic for the rename op. --- compiler/kernel/op.go | 16 +++- compiler/semantic/op.go | 35 ++++--- compiler/ztests/dynamic-field-rename.yaml | 22 +++++ pkg/field/field.go | 25 +++++ runtime/expr/renamer.go | 106 +++++++++++++++------- 5 files changed, 155 insertions(+), 49 deletions(-) create mode 100644 compiler/ztests/dynamic-field-rename.yaml diff --git a/compiler/kernel/op.go b/compiler/kernel/op.go index ba81ef9969..8111759147 100644 --- a/compiler/kernel/op.go +++ b/compiler/kernel/op.go @@ -174,10 +174,18 @@ func (b *Builder) compileLeaf(o dag.Op, parent zbuf.Puller) (zbuf.Puller, error) putter := expr.NewPutter(b.octx.Zctx, clauses) return op.NewApplier(b.octx, parent, putter), nil case *dag.Rename: - var srcs, dsts field.List - for _, a := range v.Args { - dsts = append(dsts, a.LHS.(*dag.This).Path) - srcs = append(srcs, a.RHS.(*dag.This).Path) + var srcs, dsts []*expr.Lval + for k := range v.Args { + src, err := b.compileLval(v.Args[k].RHS) + if err != nil { + return nil, err + } + dst, err := b.compileLval(v.Args[k].LHS) + if err != nil { + return nil, err + } + srcs = append(srcs, src) + dsts = append(dsts, dst) } renamer := expr.NewRenamer(b.octx.Zctx, srcs, dsts) return op.NewApplier(b.octx, parent, renamer), nil diff --git a/compiler/semantic/op.go b/compiler/semantic/op.go index cc20581d92..a85efadf47 100644 --- a/compiler/semantic/op.go +++ b/compiler/semantic/op.go @@ -640,22 +640,35 @@ func (a *analyzer) semOp(o ast.Op, seq dag.Seq) (dag.Seq, error) { case *ast.Rename: var assignments []dag.Assignment for _, fa := range o.Args { - dst, err := a.semField(fa.LHS) + dst, err := a.semExpr(fa.LHS) if err != nil { - return nil, errors.New("rename: requires explicit field references") + return nil, fmt.Errorf("rename: %w", err) } - src, err := a.semField(fa.RHS) + if !isLval(dst) { + return nil, fmt.Errorf("rename: illegal left-hand side of assignment") + } + src, err := a.semExpr(fa.RHS) if err != nil { - return nil, errors.New("rename: requires explicit field references") + return nil, fmt.Errorf("rename: %w", err) } - if len(dst.Path) != len(src.Path) { - return nil, fmt.Errorf("rename: cannot rename %s to %s", src, dst) + if !isLval(src) { + return nil, fmt.Errorf("rename: illegal right-hand side of assignment") } - // Check that the prefixes match and, if not, report first place - // that they don't. - for i := 0; i <= len(src.Path)-2; i++ { - if src.Path[i] != dst.Path[i] { - return nil, fmt.Errorf("rename: cannot rename %s to %s (differ in %s vs %s)", src, dst, src.Path[i], dst.Path[i]) + // If both paths are static validate rename paths, otherwise this + // will be done at runtime. + srcThis, srcOk := src.(*dag.This) + dstThis, dstOk := dst.(*dag.This) + if dstOk && srcOk { + s, d := field.Path(srcThis.Path), field.Path(dstThis.Path) + if len(d) != len(s) { + return nil, fmt.Errorf("rename: cannot rename %s to %s", s, d) + } + // Check that the prefixes match and, if not, report first place + // that they don't. + for i := 0; i <= len(s)-2; i++ { + if s[i] != d[i] { + return nil, fmt.Errorf("rename: cannot rename %s to %s (differ in %s vs %s)", s, d, s[i], d[i]) + } } } assignments = append(assignments, dag.Assignment{Kind: "Assignment", LHS: dst, RHS: src}) diff --git a/compiler/ztests/dynamic-field-rename.yaml b/compiler/ztests/dynamic-field-rename.yaml new file mode 100644 index 0000000000..a12768296a --- /dev/null +++ b/compiler/ztests/dynamic-field-rename.yaml @@ -0,0 +1,22 @@ +script: | + echo '{target:"foo",src:"bar"} {target:"fool",src:"baz"}' | zq -z 'rename this[target] := src' - + echo '// ===' + echo '{target:"a",a:"bar"} {target:"b",b:"baz"}' | zq -z 'rename dst := this[target]' - + # runtime error cases + echo '// ===' + echo '{foo:"a",bar:"b"}' | zq -z 'rename this[foo]["c"] := this[bar]["d"]' - + echo '// ===' + echo '{foo:"a"}' | zq -z 'rename this[foo]["c"] := this[foo]["a"]["b"]' - + +outputs: + - name: stdout + data: | + {target:"foo",foo:"bar"} + {target:"fool",fool:"baz"} + // === + {target:"a",dst:"bar"} + {target:"b",dst:"baz"} + // === + error({message:"rename: cannot rename b.d to a.c (differ in b vs a)",on:{foo:"a",bar:"b"}}) + // === + error({message:"rename: cannot rename a.a.b to a.c",on:{foo:"a"}}) diff --git a/pkg/field/field.go b/pkg/field/field.go index 2a6a2e567f..5cf30ad9a2 100644 --- a/pkg/field/field.go +++ b/pkg/field/field.go @@ -15,6 +15,20 @@ func (p Path) String() string { return strings.Join(p, ".") } +// AppendTo appends the string representation of the path to byte slice b. +func (p Path) AppendTo(b []byte) []byte { + if len(p) == 0 { + return append(b, "this"...) + } + for i, s := range p { + if i > 0 { + b = append(b, '.') + } + b = append(b, s...) + } + return b +} + func (p Path) Leaf() string { return p[len(p)-1] } @@ -65,6 +79,17 @@ func (l List) String() string { return strings.Join(paths, ",") } +// AppendTo appends the string representation of the list to byte slice b. +func (l List) AppendTo(b []byte) []byte { + for i, p := range l { + if i > 0 { + b = append(b, ',') + } + b = p.AppendTo(b) + } + return b +} + func (l List) Has(in Path) bool { return slices.ContainsFunc(l, in.Equal) } diff --git a/runtime/expr/renamer.go b/runtime/expr/renamer.go index 2735f0a751..cca4b79394 100644 --- a/runtime/expr/renamer.go +++ b/runtime/expr/renamer.go @@ -17,13 +17,80 @@ type Renamer struct { zctx *zed.Context // For the dst field name, we just store the leaf name since the // src path and the dst path are the same and only differ in the leaf name. - srcs field.List - dsts field.List - typeMap map[int]*zed.TypeRecord + srcs []*Lval + dsts []*Lval + typeMap map[int]map[string]*zed.TypeRecord + // fieldsStr is used to reduce allocations when computing the fields id. + fieldsStr []byte } -func NewRenamer(zctx *zed.Context, srcs, dsts field.List) *Renamer { - return &Renamer{zctx, srcs, dsts, make(map[int]*zed.TypeRecord)} +func NewRenamer(zctx *zed.Context, srcs, dsts []*Lval) *Renamer { + return &Renamer{zctx, srcs, dsts, make(map[int]map[string]*zed.TypeRecord), nil} +} + +func (r *Renamer) Eval(ectx Context, this *zed.Value) *zed.Value { + if !zed.IsRecordType(this.Type) { + return this + } + srcs, dsts, err := r.evalFields(ectx, this) + if err != nil { + return ectx.CopyValue(*r.zctx.WrapError(fmt.Sprintf("rename: %s", err), this)) + } + id := this.Type.ID() + m, ok := r.typeMap[id] + if !ok { + m = make(map[string]*zed.TypeRecord) + r.typeMap[id] = m + } + r.fieldsStr = srcs.AppendTo(r.fieldsStr[:0]) + r.fieldsStr = dsts.AppendTo(r.fieldsStr) + typ, ok := m[string(r.fieldsStr)] + if !ok { + var err error + typ, err = r.computeType(zed.TypeRecordOf(this.Type), srcs, dsts) + if err != nil { + return ectx.CopyValue(*r.zctx.WrapError(fmt.Sprintf("rename: %s", err), this)) + } + m[string(r.fieldsStr)] = typ + } + out := this.Copy() + return ectx.NewValue(typ, out.Bytes()) +} + +func (r *Renamer) evalFields(ectx Context, this *zed.Value) (field.List, field.List, error) { + var srcs, dsts field.List + for i := range r.srcs { + src, err := r.srcs[i].Eval(ectx, this) + if err != nil { + return nil, nil, err + } + dst, err := r.dsts[i].Eval(ectx, this) + if err != nil { + return nil, nil, err + } + if len(src) != len(dst) { + return nil, nil, fmt.Errorf("cannot rename %s to %s", src, dst) + } + for i := 0; i <= len(src)-2; i++ { + if src[i] != dst[i] { + return nil, nil, fmt.Errorf("cannot rename %s to %s (differ in %s vs %s)", src, dst, src[i], dst[i]) + } + } + srcs = append(srcs, src) + dsts = append(dsts, dst) + } + return srcs, dsts, nil +} + +func (r *Renamer) computeType(typ *zed.TypeRecord, srcs, dsts field.List) (*zed.TypeRecord, error) { + for k, dst := range dsts { + var err error + typ, err = r.dstType(typ, srcs[k], dst) + if err != nil { + return nil, err + } + } + return typ, nil } func (r *Renamer) dstType(typ *zed.TypeRecord, src, dst field.Path) (*zed.TypeRecord, error) { @@ -57,32 +124,3 @@ func (r *Renamer) dstType(typ *zed.TypeRecord, src, dst field.Path) (*zed.TypeRe } return typ, nil } - -func (r *Renamer) computeType(typ *zed.TypeRecord) (*zed.TypeRecord, error) { - for k, dst := range r.dsts { - var err error - typ, err = r.dstType(typ, r.srcs[k], dst) - if err != nil { - return nil, err - } - } - return typ, nil -} - -func (r *Renamer) Eval(ectx Context, this *zed.Value) *zed.Value { - if !zed.IsRecordType(this.Type) { - return this - } - id := this.Type.ID() - typ, ok := r.typeMap[id] - if !ok { - var err error - typ, err = r.computeType(zed.TypeRecordOf(this.Type)) - if err != nil { - return r.zctx.WrapError(fmt.Sprintf("rename: %s", err), this) - } - r.typeMap[id] = typ - } - out := this.Copy() - return ectx.NewValue(typ, out.Bytes()) -}