From 09f1fb5449d37fa9484631dbaa8f834609fd5004 Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Tue, 24 Sep 2024 18:50:32 -0700 Subject: [PATCH] change variant to dynamic This commit changes our terminology for a top-level dynamic type to "Dynamic" from "Variant". The term "variant" has been corrupted by the industry to mean efficiently-encoded semi-structured data when the traditional meaning from programming languages is a tagged union. To avoid this confusion, we're following clickhouse's terminology and using the term dynamic instead. This change is internal to the code only and does not affect any user-facing documentation as the super-structured data model is strongly typed throughout and does not expose the implied top-level dynamic type as a user-visible type. --- runtime/vam/expr/function/bytes.go | 8 +-- runtime/vam/expr/function/string.go | 2 +- runtime/vam/expr/index.go | 2 +- runtime/vam/expr/logic.go | 10 ++-- runtime/vam/materialize.go | 8 +-- runtime/vam/op/agg.go | 4 +- runtime/vam/op/yield.go | 4 +- runtime/vam/op/ztests/arith.yaml | 2 +- runtime/vam/op/ztests/compare.yaml | 2 +- runtime/vcache/loader.go | 6 +- runtime/vcache/project.go | 8 +-- runtime/vcache/shadow.go | 12 ++-- vector/any.go | 8 +-- vector/apply.go | 28 ++++----- vector/bool.go | 2 +- vector/dynamic.go | 47 +++++++++++++++ vector/int.go | 2 +- vector/tagmap.go | 2 +- vector/union.go | 8 +-- vector/variant.go | 47 --------------- vector/view.go | 8 +-- vng/{variant.go => dynamic.go} | 88 ++++++++++++++--------------- vng/metadata.go | 14 ++--- vng/writer.go | 10 ++-- 24 files changed, 166 insertions(+), 166 deletions(-) create mode 100644 vector/dynamic.go delete mode 100644 vector/variant.go rename vng/{variant.go => dynamic.go} (53%) diff --git a/runtime/vam/expr/function/bytes.go b/runtime/vam/expr/function/bytes.go index fe3b22cd81..7f1b19a89a 100644 --- a/runtime/vam/expr/function/bytes.go +++ b/runtime/vam/expr/function/bytes.go @@ -30,7 +30,7 @@ func (b *Base64) Call(args ...vector.Any) vector.Any { out.Append(base64.StdEncoding.EncodeToString(bytes)) } err := vector.NewStringError(b.zctx, "base64: illegal null argument", errcnt) - return vector.NewVariant(tags, []vector.Any{out, err}) + return vector.NewDynamic(tags, []vector.Any{out, err}) case zed.IDString: errvals := vector.NewStringEmpty(0, nil) tags := make([]uint32, val.Len()) @@ -49,7 +49,7 @@ func (b *Base64) Call(args ...vector.Any) vector.Any { out.Append(bytes) } err := vector.NewWrappedError(b.zctx, "base64: string argument is not base64", errvals) - return vector.NewVariant(tags, []vector.Any{out, err}) + return vector.NewDynamic(tags, []vector.Any{out, err}) default: return vector.NewWrappedError(b.zctx, "base64: argument must a bytes or string type", val) } @@ -77,7 +77,7 @@ func (h *Hex) Call(args ...vector.Any) vector.Any { out.Append(hex.EncodeToString(bytes)) } err := vector.NewStringError(h.zctx, "hex: illegal null argument", errcnt) - return vector.NewVariant(tags, []vector.Any{out, err}) + return vector.NewDynamic(tags, []vector.Any{out, err}) case zed.IDString: errvals := vector.NewStringEmpty(0, nil) tags := make([]uint32, val.Len()) @@ -96,7 +96,7 @@ func (h *Hex) Call(args ...vector.Any) vector.Any { out.Append(bytes) } err := vector.NewWrappedError(h.zctx, "hex: string argument is not hexidecimal", errvals) - return vector.NewVariant(tags, []vector.Any{out, err}) + return vector.NewDynamic(tags, []vector.Any{out, err}) default: return vector.NewWrappedError(h.zctx, "hex: argument must a bytes or string type", val) } diff --git a/runtime/vam/expr/function/string.go b/runtime/vam/expr/function/string.go index 9cad4ffa8e..77e48fbab9 100644 --- a/runtime/vam/expr/function/string.go +++ b/runtime/vam/expr/function/string.go @@ -105,7 +105,7 @@ func (r *Replace) Call(args ...vector.Any) vector.Any { out.Append(strings.ReplaceAll(s, old, new)) } errval := vector.NewStringError(r.zctx, "replace: an input arg is null", errcnt) - return vector.NewVariant(tags, []vector.Any{out, errval}) + return vector.NewDynamic(tags, []vector.Any{out, errval}) } // https://github.com/brimdata/zed/blob/main/docs/language/functions.md#run_len diff --git a/runtime/vam/expr/index.go b/runtime/vam/expr/index.go index fd03f6d32b..ce81839031 100644 --- a/runtime/vam/expr/index.go +++ b/runtime/vam/expr/index.go @@ -92,5 +92,5 @@ func indexRecord(zctx *zed.Context, record *vector.Record, index vector.Any) vec for i, field := range record.Fields { out[i] = vector.NewView(viewIndexes[i], field) } - return vector.NewVariant(tags, out) + return vector.NewDynamic(tags, out) } diff --git a/runtime/vam/expr/logic.go b/runtime/vam/expr/logic.go index a1fe8c614e..5a2fd120eb 100644 --- a/runtime/vam/expr/logic.go +++ b/runtime/vam/expr/logic.go @@ -50,7 +50,7 @@ func NewLogicalOr(zctx *zed.Context, lhs, rhs Evaluator) *Or { } func (a *And) Eval(val vector.Any) vector.Any { - //XXX change this logic to handle variant instead of simple ok decision, + //XXX change this logic to handle dynamic instead of simple ok decision, // if there are any valid bools then we need to and them together lhs, ok := EvalBool(a.zctx, val, a.lhs) if !ok { @@ -102,15 +102,15 @@ func (o *Or) Eval(val vector.Any) vector.Any { // slot and they are returned as an error. If all of the value slots are errors, // then the return value is nil. func EvalBool(zctx *zed.Context, val vector.Any, e Evaluator) (vector.Any, bool) { - //XXX Eval could return a variant of errors and bools and we should + //XXX Eval could return a dynamic vector of errors and bools and we should // handle this correctly so the logic above is really the fast path - // and a slower path will handle picking apart the variant. - // maybe we could have a generic way to traverse variants for + // and a slower path will handle picking apart the dynamic vector. + // maybe we could have a generic way to traverse dynamics for // appliers doing their thing along the slow path if val, ok := vector.Under(e.Eval(val)).(*vector.Bool); ok { return val, true } - //XXX need to implement a sparse variant (vector.Collection?) + //XXX need to implement a sparse dynamic (vector.Collection?) // and check for that here. // for now, if the vector is not uniformly boolean, we return error. // XXX example is a field ref a union of structs where the type of diff --git a/runtime/vam/materialize.go b/runtime/vam/materialize.go index 32a399e947..3d44c924ba 100644 --- a/runtime/vam/materialize.go +++ b/runtime/vam/materialize.go @@ -26,9 +26,9 @@ func (m *Materializer) Pull(done bool) (zbuf.Batch, error) { if vec == nil || err != nil { return nil, err } - variant, _ := vec.(*vector.Variant) + d, _ := vec.(*vector.Dynamic) var typ zed.Type - if variant == nil { + if d == nil { typ = vec.Type() } builder := zcode.NewBuilder() @@ -36,8 +36,8 @@ func (m *Materializer) Pull(done bool) (zbuf.Batch, error) { n := vec.Len() for slot := uint32(0); slot < n; slot++ { vec.Serialize(builder, slot) - if variant != nil { - typ = variant.TypeOf(slot) + if d != nil { + typ = d.TypeOf(slot) } val := zed.NewValue(typ, bytes.Clone(builder.Bytes().Body())) vals = append(vals, val) diff --git a/runtime/vam/op/agg.go b/runtime/vam/op/agg.go index b7e4c0cf35..4c2cbe26c5 100644 --- a/runtime/vam/op/agg.go +++ b/runtime/vam/op/agg.go @@ -53,7 +53,7 @@ func (c *CountByString) Pull(done bool) (vector.Any, error) { } func (c *CountByString) update(val vector.Any) { - if val, ok := val.(*vector.Variant); ok { + if val, ok := val.(*vector.Dynamic); ok { for _, val := range val.Values { c.update(val) } @@ -183,7 +183,7 @@ func (c *Sum) Pull(done bool) (vector.Any, error) { } func (c *Sum) update(vec vector.Any) { - if vec, ok := vec.(*vector.Variant); ok { + if vec, ok := vec.(*vector.Dynamic); ok { for _, vec := range vec.Values { c.update(vec) } diff --git a/runtime/vam/op/yield.go b/runtime/vam/op/yield.go index dcebab45b1..ee337279c5 100644 --- a/runtime/vam/op/yield.go +++ b/runtime/vam/op/yield.go @@ -49,7 +49,7 @@ func filterQuiet(val vector.Any) vector.Any { return val } -// XXX should work for input variants +// XXX should work for vector.Dynamic func interleave(vals []vector.Any) vector.Any { if len(vals) < 2 { panic("interleave requires two or more vals") @@ -61,5 +61,5 @@ func interleave(vals []vector.Any) vector.Any { tags[k] = k % nvals } - return vector.NewVariant(tags, vals) + return vector.NewDynamic(tags, vals) } diff --git a/runtime/vam/op/ztests/arith.yaml b/runtime/vam/op/ztests/arith.yaml index c647c2d9e3..69cd65782e 100644 --- a/runtime/vam/op/ztests/arith.yaml +++ b/runtime/vam/op/ztests/arith.yaml @@ -1,4 +1,4 @@ -# Test that arithmetic works on union and variant vectors. +# Test that arithmetic works on union and dynamic vectors. script: | zq -o t.vng -f vng - diff --git a/runtime/vam/op/ztests/compare.yaml b/runtime/vam/op/ztests/compare.yaml index a7661877c3..7f9c9a89e6 100644 --- a/runtime/vam/op/ztests/compare.yaml +++ b/runtime/vam/op/ztests/compare.yaml @@ -1,4 +1,4 @@ -# Test that comparison works on union and variant vectors. +# Test that comparison works on union and dynamic vectors. script: | zq -o t.vng -f vng - diff --git a/runtime/vcache/loader.go b/runtime/vcache/loader.go index 8924235cc7..b3346f05dc 100644 --- a/runtime/vcache/loader.go +++ b/runtime/vcache/loader.go @@ -59,7 +59,7 @@ func (l *loader) load(paths Path, s shadow) (vector.Any, error) { func (l *loader) loadVector(g *errgroup.Group, paths Path, s shadow) { switch s := s.(type) { - case *variant: + case *dynamic: //XXX we need an ordered option to load tags only when needed l.loadUint32(g, &s.mu, &s.tags, s.loc) for _, m := range s.vals { @@ -476,7 +476,7 @@ func (l *loader) loadOffsets(g *errgroup.Group, mu *sync.Mutex, slice *[]uint32, func (l *loader) fetchNulls(g *errgroup.Group, paths Path, s shadow) { switch s := s.(type) { - case *variant: + case *dynamic: for _, m := range s.vals { l.fetchNulls(g, paths, m) } @@ -533,7 +533,7 @@ func (l *loader) fetchNulls(g *errgroup.Group, paths Path, s shadow) { func flattenNulls(paths Path, s shadow, parent *vector.Bool) { switch s := s.(type) { - case *variant: + case *dynamic: for _, m := range s.vals { flattenNulls(paths, m, nil) } diff --git a/runtime/vcache/project.go b/runtime/vcache/project.go index b3f1f7e955..f5587288e2 100644 --- a/runtime/vcache/project.go +++ b/runtime/vcache/project.go @@ -9,8 +9,8 @@ import ( func project(zctx *zed.Context, paths Path, s shadow) vector.Any { switch s := s.(type) { - case *variant: - return projectVariant(zctx, paths, s) + case *dynamic: + return projectDynamic(zctx, paths, s) case *record: return projectRecord(zctx, paths, s) case *array: @@ -54,12 +54,12 @@ func project(zctx *zed.Context, paths Path, s shadow) vector.Any { } } -func projectVariant(zctx *zed.Context, paths Path, s *variant) vector.Any { +func projectDynamic(zctx *zed.Context, paths Path, s *dynamic) vector.Any { vals := make([]vector.Any, 0, len(s.vals)) for _, m := range s.vals { vals = append(vals, project(zctx, paths, m)) } - return vector.NewVariant(s.tags, vals) + return vector.NewDynamic(s.tags, vals) } func projectRecord(zctx *zed.Context, paths Path, s *record) vector.Any { diff --git a/runtime/vcache/shadow.go b/runtime/vcache/shadow.go index 3fdad45ce5..6077c02bc6 100644 --- a/runtime/vcache/shadow.go +++ b/runtime/vcache/shadow.go @@ -21,16 +21,16 @@ type shadow interface { length() uint32 } -type variant struct { +type dynamic struct { mu sync.Mutex len uint32 - tags []uint32 // need not be loaded for unordered variants + tags []uint32 // need not be loaded for unordered dynamics loc vng.Segment vals []shadow } -func (v *variant) length() uint32 { - return v.len +func (d *dynamic) length() uint32 { + return d.len } type record struct { @@ -133,12 +133,12 @@ func (c count) length() uint32 { // by the runtime. func newShadow(m vng.Metadata, n *vng.Nulls, nullsCnt uint32) shadow { switch m := m.(type) { - case *vng.Variant: + case *vng.Dynamic: vals := make([]shadow, 0, len(m.Values)) for _, val := range m.Values { vals = append(vals, newShadow(val, nil, 0)) } - return &variant{ + return &dynamic{ vals: vals, len: m.Len(), loc: m.Tags, diff --git a/vector/any.go b/vector/any.go index 6c9e63c1fc..10e04c7727 100644 --- a/vector/any.go +++ b/vector/any.go @@ -25,9 +25,9 @@ type Builder func(*zcode.Builder) bool func Combine(vec Any, index []uint32, add Any) Any { var vecs []Any tags := make([]uint32, int(vec.Len())+len(index)) - if variant, ok := vec.(*Variant); ok { - vecs = variant.Values - varTags := variant.Tags + if d, ok := vec.(*Dynamic); ok { + vecs = d.Values + varTags := d.Tags n := uint32(len(vecs)) for i := uint32(0); i < uint32(len(tags)); i++ { if len(index) > 0 && i == index[0] { @@ -44,5 +44,5 @@ func Combine(vec Any, index []uint32, add Any) Any { tags[k] = 1 } } - return NewVariant(tags, append(vecs, add)) + return NewDynamic(tags, append(vecs, add)) } diff --git a/vector/apply.go b/vector/apply.go index 172215185b..b74596c40a 100644 --- a/vector/apply.go +++ b/vector/apply.go @@ -1,45 +1,45 @@ package vector -// Apply applies eval to vecs. If any element of vecs is a Variant, Apply rips +// Apply applies eval to vecs. If any element of vecs is a Dynamic, Apply rips // vecs accordingly, applies eval to the ripped vectors, and stitches the -// results together into a Variant. If ripUnions is true, Apply also rips +// results together into a Dynamic. If ripUnions is true, Apply also rips // Unions. func Apply(ripUnions bool, eval func(...Any) Any, vecs ...Any) Any { if ripUnions { for k, vec := range vecs { if union, ok := Under(vec).(*Union); ok { - vecs[k] = union.Variant + vecs[k] = union.Dynamic } } } - variant, ok := findVariant(vecs) + d, ok := findDynamic(vecs) if !ok { return eval(vecs...) } var results []Any - for _, ripped := range rip(vecs, variant) { + for _, ripped := range rip(vecs, d) { results = append(results, Apply(ripUnions, eval, ripped...)) } - // Stitch results together by creating a Variant. - return NewVariant(variant.Tags, results) + // Stitch results together in a new Dynamic. + return NewDynamic(d.Tags, results) } -func findVariant(vecs []Any) (*Variant, bool) { +func findDynamic(vecs []Any) (*Dynamic, bool) { for _, vec := range vecs { - if variant, ok := vec.(*Variant); ok { - return variant, true + if d, ok := vec.(*Dynamic); ok { + return d, true } } return nil, false } -func rip(vecs []Any, variant *Variant) [][]Any { +func rip(vecs []Any, d *Dynamic) [][]Any { var ripped [][]Any - for j, rev := range variant.TagMap.Reverse { + for j, rev := range d.TagMap.Reverse { var newVecs []Any for _, vec := range vecs { - if vec == variant { - newVecs = append(newVecs, variant.Values[j]) + if vec == d { + newVecs = append(newVecs, d.Values[j]) } else { newVecs = append(newVecs, NewView(rev, vec)) } diff --git a/vector/bool.go b/vector/bool.go index f0c52a10ab..152c679621 100644 --- a/vector/bool.go +++ b/vector/bool.go @@ -81,7 +81,7 @@ func BoolValue(vec Any, slot uint32) bool { return vec.Value().Ptr().AsBool() case *Dict: return BoolValue(vec.Any, uint32(vec.Index[slot])) - case *Variant: + case *Dynamic: tag := vec.Tags[slot] return BoolValue(vec.Values[tag], vec.TagMap.Forward[slot]) case *View: diff --git a/vector/dynamic.go b/vector/dynamic.go new file mode 100644 index 0000000000..7c7205c55c --- /dev/null +++ b/vector/dynamic.go @@ -0,0 +1,47 @@ +package vector + +import ( + "github.com/brimdata/zed" + "github.com/brimdata/zed/zcode" +) + +// Dynamic is an ordered sequence of values taken from one or more +// hetereogenously-typed vectors. +type Dynamic struct { + Tags []uint32 + Values []Any + TagMap *TagMap +} + +var _ Any = (*Dynamic)(nil) + +func NewDynamic(tags []uint32, values []Any) *Dynamic { + return &Dynamic{Tags: tags, Values: values, TagMap: NewTagMap(tags, values)} +} + +func (*Dynamic) Type() zed.Type { + panic("can't call Type() on a vector.Dynamic") +} + +func (d *Dynamic) TypeOf(slot uint32) zed.Type { + vals := d.Values[d.Tags[slot]] + if v2, ok := vals.(*Dynamic); ok { + return v2.TypeOf(d.TagMap.Forward[slot]) + } + return vals.Type() +} + +func (d *Dynamic) Len() uint32 { + if d.Tags != nil { + return uint32(len(d.Tags)) + } + var length uint32 + for _, val := range d.Values { + length += val.Len() + } + return length +} + +func (d *Dynamic) Serialize(b *zcode.Builder, slot uint32) { + d.Values[d.Tags[slot]].Serialize(b, d.TagMap.Forward[slot]) +} diff --git a/vector/int.go b/vector/int.go index 7fdcebbb79..40d20dd332 100644 --- a/vector/int.go +++ b/vector/int.go @@ -58,7 +58,7 @@ func IntValue(vec Any, slot uint32) (int64, bool) { return vec.Value().Ptr().AsInt(), vec.Nulls.Value(slot) case *Dict: return IntValue(vec.Any, uint32(vec.Index[slot])) - case *Variant: + case *Dynamic: tag := vec.Tags[slot] return IntValue(vec.Values[tag], vec.TagMap.Forward[slot]) case *View: diff --git a/vector/tagmap.go b/vector/tagmap.go index b69c5a7772..892d2dddab 100644 --- a/vector/tagmap.go +++ b/vector/tagmap.go @@ -1,6 +1,6 @@ package vector -// TagMap is used by variants and unions to map slots between parent and child in +// TagMap is used by dynamics and unions to map slots between parent and child in // both the forward and reverse directions. We need this because vectors are stored // in a dense format where different types hold only the values needed for that type. // If we stored vectors in a sparse format, the amount of overhead would increase diff --git a/vector/union.go b/vector/union.go index 16f12e5738..61a7abc718 100644 --- a/vector/union.go +++ b/vector/union.go @@ -6,7 +6,7 @@ import ( ) type Union struct { - *Variant + *Dynamic Typ *zed.TypeUnion Nulls *Bool } @@ -14,7 +14,7 @@ type Union struct { var _ Any = (*Union)(nil) func NewUnion(typ *zed.TypeUnion, tags []uint32, vals []Any, nulls *Bool) *Union { - return &Union{NewVariant(tags, vals), typ, nulls} + return &Union{NewDynamic(tags, vals), typ, nulls} } func (u *Union) Type() zed.Type { @@ -24,14 +24,14 @@ func (u *Union) Type() zed.Type { func (u *Union) Serialize(b *zcode.Builder, slot uint32) { b.BeginContainer() b.Append(zed.EncodeInt(int64(u.Tags[slot]))) - u.Variant.Serialize(b, slot) + u.Dynamic.Serialize(b, slot) b.EndContainer() } func Deunion(vec Any) Any { if union, ok := vec.(*Union); ok { // XXX if the Union has Nulls this will be broken. - return union.Variant + return union.Dynamic } return vec } diff --git a/vector/variant.go b/vector/variant.go deleted file mode 100644 index c447777616..0000000000 --- a/vector/variant.go +++ /dev/null @@ -1,47 +0,0 @@ -package vector - -import ( - "github.com/brimdata/zed" - "github.com/brimdata/zed/zcode" -) - -// Variant is an ordered sequence of values taken from one or more -// hetereogenously-typed vectors. -type Variant struct { - Tags []uint32 - Values []Any - TagMap *TagMap -} - -var _ Any = (*Variant)(nil) - -func NewVariant(tags []uint32, values []Any) *Variant { - return &Variant{Tags: tags, Values: values, TagMap: NewTagMap(tags, values)} -} - -func (v *Variant) Type() zed.Type { - panic("can't call Type() on a vector.Variant") -} - -func (v *Variant) TypeOf(slot uint32) zed.Type { - vals := v.Values[v.Tags[slot]] - if v2, ok := vals.(*Variant); ok { - return v2.TypeOf(v.TagMap.Forward[slot]) - } - return vals.Type() -} - -func (v *Variant) Len() uint32 { - if v.Tags != nil { - return uint32(len(v.Tags)) - } - var length uint32 - for _, val := range v.Values { - length += val.Len() - } - return length -} - -func (v *Variant) Serialize(b *zcode.Builder, slot uint32) { - v.Values[v.Tags[slot]].Serialize(b, v.TagMap.Forward[slot]) -} diff --git a/vector/view.go b/vector/view.go index 92163f1c92..b42936ae8b 100644 --- a/vector/view.go +++ b/vector/view.go @@ -24,10 +24,10 @@ func NewView(index []uint32, val Any) Any { } return NewDict(val.Any, index2, nil, nulls) case *Union: - tags, values := viewForUnionOrVariant(index, val.Tags, val.TagMap.Forward, val.Values) + tags, values := viewForUnionOrDynamic(index, val.Tags, val.TagMap.Forward, val.Values) return NewUnion(val.Typ, tags, values, nil) - case *Variant: - return NewVariant(viewForUnionOrVariant(index, val.Tags, val.TagMap.Forward, val.Values)) + case *Dynamic: + return NewDynamic(viewForUnionOrDynamic(index, val.Tags, val.TagMap.Forward, val.Values)) case *View: index2 := make([]uint32, len(index)) for k, idx := range index { @@ -38,7 +38,7 @@ func NewView(index []uint32, val Any) Any { return &View{val, index} } -func viewForUnionOrVariant(index, tags, forward []uint32, values []Any) ([]uint32, []Any) { +func viewForUnionOrDynamic(index, tags, forward []uint32, values []Any) ([]uint32, []Any) { indexes := make([][]uint32, len(values)) resultTags := make([]uint32, len(index)) for k, index := range index { diff --git a/vng/variant.go b/vng/dynamic.go similarity index 53% rename from vng/variant.go rename to vng/dynamic.go index 5ddfcc63d2..c1e4f4289f 100644 --- a/vng/variant.go +++ b/vng/dynamic.go @@ -10,76 +10,76 @@ import ( "golang.org/x/sync/errgroup" ) -type VariantEncoder struct { +type DynamicEncoder struct { tags *Int64Encoder values []Encoder which map[zed.Type]int len uint32 } -var _ zio.Writer = (*VariantEncoder)(nil) +var _ zio.Writer = (*DynamicEncoder)(nil) -func NewVariantEncoder() *VariantEncoder { - return &VariantEncoder{ +func NewDynamicEncoder() *DynamicEncoder { + return &DynamicEncoder{ tags: NewInt64Encoder(), which: make(map[zed.Type]int), } } -// The variant encoder self-organizes around the types that are +// The dynamic encoder self-organizes around the types that are // written to it. No need to define the schema up front! // We track the types seen first-come, first-served and the // VNG metadata structure follows accordingly. -func (v *VariantEncoder) Write(val zed.Value) error { +func (d *DynamicEncoder) Write(val zed.Value) error { typ := val.Type() - tag, ok := v.which[typ] + tag, ok := d.which[typ] if !ok { - tag = len(v.values) - v.values = append(v.values, NewEncoder(typ)) - v.which[typ] = tag + tag = len(d.values) + d.values = append(d.values, NewEncoder(typ)) + d.which[typ] = tag } - v.tags.Write(int64(tag)) - v.len++ - v.values[tag].Write(val.Bytes()) + d.tags.Write(int64(tag)) + d.len++ + d.values[tag].Write(val.Bytes()) return nil } -func (v *VariantEncoder) Encode() (Metadata, uint64, error) { +func (d *DynamicEncoder) Encode() (Metadata, uint64, error) { var group errgroup.Group - if len(v.values) > 1 { - v.tags.Encode(&group) + if len(d.values) > 1 { + d.tags.Encode(&group) } - for _, val := range v.values { + for _, val := range d.values { val.Encode(&group) } if err := group.Wait(); err != nil { return nil, 0, err } - if len(v.values) == 1 { - off, meta := v.values[0].Metadata(0) + if len(d.values) == 1 { + off, meta := d.values[0].Metadata(0) return meta, off, nil } - values := make([]Metadata, 0, len(v.values)) - off, tags := v.tags.Metadata(0) - for _, val := range v.values { + values := make([]Metadata, 0, len(d.values)) + off, tags := d.tags.Metadata(0) + for _, val := range d.values { var meta Metadata off, meta = val.Metadata(off) values = append(values, meta) } - return &Variant{ + return &Dynamic{ Tags: tags.(*Primitive).Location, Values: values, - Length: v.len, + Length: d.len, }, off, nil } -func (v *VariantEncoder) Emit(w io.Writer) error { - if len(v.values) > 1 { - if err := v.tags.Emit(w); err != nil { +func (d *DynamicEncoder) Emit(w io.Writer) error { + if len(d.values) > 1 { + if err := d.tags.Emit(w); err != nil { return err } } - for _, value := range v.values { + for _, value := range d.values { if err := value.Emit(w); err != nil { return err } @@ -87,17 +87,17 @@ func (v *VariantEncoder) Emit(w io.Writer) error { return nil } -type variantBuilder struct { +type dynamicBuilder struct { types []zed.Type tags *Int64Decoder values []Builder builder *zcode.Builder } -func newVariantBuilder(zctx *zed.Context, variant *Variant, reader io.ReaderAt) (*variantBuilder, error) { - values := make([]Builder, 0, len(variant.Values)) - types := make([]zed.Type, 0, len(variant.Values)) - for _, val := range variant.Values { +func newDynamicBuilder(zctx *zed.Context, d *Dynamic, reader io.ReaderAt) (*dynamicBuilder, error) { + values := make([]Builder, 0, len(d.Values)) + types := make([]zed.Type, 0, len(d.Values)) + for _, val := range d.Values { r, err := NewBuilder(val, reader) if err != nil { return nil, err @@ -105,36 +105,36 @@ func newVariantBuilder(zctx *zed.Context, variant *Variant, reader io.ReaderAt) values = append(values, r) types = append(types, val.Type(zctx)) } - return &variantBuilder{ + return &dynamicBuilder{ types: types, - tags: NewInt64Decoder(variant.Tags, reader), + tags: NewInt64Decoder(d.Tags, reader), values: values, builder: zcode.NewBuilder(), }, nil } -func (v *variantBuilder) Read() (*zed.Value, error) { - b := v.builder +func (d *dynamicBuilder) Read() (*zed.Value, error) { + b := d.builder b.Truncate() - tag, err := v.tags.Next() + tag, err := d.tags.Next() if err != nil { if err == io.EOF { err = nil } return nil, err } - if int(tag) >= len(v.types) { - return nil, fmt.Errorf("bad tag encountered scanning VNG variant: tag %d when only %d types", tag, len(v.types)) + if int(tag) >= len(d.types) { + return nil, fmt.Errorf("bad tag encountered scanning VNG dynamic: tag %d when only %d types", tag, len(d.types)) } - if err := v.values[tag].Build(b); err != nil { + if err := d.values[tag].Build(b); err != nil { return nil, err } - return zed.NewValue(v.types[tag], b.Bytes().Body()).Ptr(), nil + return zed.NewValue(d.types[tag], b.Bytes().Body()).Ptr(), nil } func NewZedReader(zctx *zed.Context, meta Metadata, r io.ReaderAt) (zio.Reader, error) { - if variant, ok := meta.(*Variant); ok { - return newVariantBuilder(zctx, variant, r) + if d, ok := meta.(*Dynamic); ok { + return newDynamicBuilder(zctx, d, r) } values, err := NewBuilder(meta, r) if err != nil { diff --git a/vng/metadata.go b/vng/metadata.go index e7c90f1f80..c755326059 100644 --- a/vng/metadata.go +++ b/vng/metadata.go @@ -208,20 +208,20 @@ func (c *Const) Len() uint32 { return c.Count } -type Variant struct { +type Dynamic struct { Tags Segment Values []Metadata Length uint32 } -var _ Metadata = (*Variant)(nil) +var _ Metadata = (*Dynamic)(nil) -func (*Variant) Type(zctx *zed.Context) zed.Type { - panic("Type should not be called on Variant") +func (*Dynamic) Type(zctx *zed.Context) zed.Type { + panic("Type should not be called on Dynamic") } -func (v *Variant) Len() uint32 { - return v.Length +func (d *Dynamic) Len() uint32 { + return d.Length } var Template = []interface{}{ @@ -235,5 +235,5 @@ var Template = []interface{}{ Error{}, Nulls{}, Const{}, - Variant{}, + Dynamic{}, } diff --git a/vng/writer.go b/vng/writer.go index 11d6f5c497..aad813a37d 100644 --- a/vng/writer.go +++ b/vng/writer.go @@ -16,7 +16,7 @@ import ( type Writer struct { zctx *zed.Context writer io.WriteCloser - variant *VariantEncoder + dynamic *DynamicEncoder } var _ zio.Writer = (*Writer)(nil) @@ -25,7 +25,7 @@ func NewWriter(w io.WriteCloser) *Writer { return &Writer{ zctx: zed.NewContext(), writer: w, - variant: NewVariantEncoder(), + dynamic: NewDynamicEncoder(), } } @@ -38,11 +38,11 @@ func (w *Writer) Close() error { } func (w *Writer) Write(val zed.Value) error { - return w.variant.Write(val) + return w.dynamic.Write(val) } func (w *Writer) finalize() error { - meta, dataSize, err := w.variant.Encode() + meta, dataSize, err := w.dynamic.Encode() if err != nil { return fmt.Errorf("system error: could not encode VNG metadata: %w", err) } @@ -71,7 +71,7 @@ func (w *Writer) finalize() error { return fmt.Errorf("system error: could not write VNG metadata section: %w", err) } // Data section - if err := w.variant.Emit(w.writer); err != nil { + if err := w.dynamic.Emit(w.writer); err != nil { return fmt.Errorf("system error: could not write VNG data section: %w", err) } return nil