vng: Drop builder (#5557)

This pr removes the builder interface from package vng. Instead when loading CSUP files into the sequential runtime the file is loaded as a vector then serialized into row format.
brimdata · Jan 8, 2025 · dce22e5 · dce22e5
1 parent a0d6d4d
commit dce22e5
Show file tree

Hide file tree

Showing 14 changed files with 68 additions and 530 deletions.
diff --git a/runtime/vcache/loader.go b/runtime/vcache/loader.go
@@ -441,7 +441,7 @@ func (l *loader) loadUint32(g *errgroup.Group, mu *sync.Mutex, slice *[]uint32,
 		if *slice != nil {
 			return nil
 		}
-		v, err := vng.ReadUint32Vector(loc, l.r)
+		v, err := vng.ReadUint32s(loc, l.r)
 		if err != nil {
 			return err
 		}
@@ -463,7 +463,7 @@ func (l *loader) loadOffsets(g *errgroup.Group, mu *sync.Mutex, slice *[]uint32,
 		if *slice != nil {
 			return nil
 		}
-		v, err := vng.ReadUint32Vector(loc, l.r)
+		v, err := vng.ReadUint32s(loc, l.r)
 		if err != nil {
 			return err
 		}

diff --git a/runtime/vcache/nulls.go b/runtime/vcache/nulls.go
@@ -34,28 +34,24 @@ func (n *nulls) fetch(g *errgroup.Group, reader io.ReaderAt) {
 		}
 		length := n.meta.Count + n.meta.Values.Len()
 		n.local = vector.NewBoolEmpty(length, nil)
-		runlens := vng.NewInt64Decoder(n.meta.Runs, reader) //XXX 32-bit reader?
+		runlens, err := vng.ReadUint32s(n.meta.Runs, reader)
+		if err != nil {
+			return err
+		}
 		var null bool
-		var off int
+		var off uint32
 		b := n.local
-		for {
-			run, err := runlens.Next()
-			if err != nil {
-				if err == io.EOF {
-					n.meta = nil
-					err = nil
-				}
-				return err
-			}
+		for _, run := range runlens {
 			if null {
-				for i := 0; int64(i) < run; i++ {
-					slot := uint32(off + i)
+				for i := range run {
+					slot := off + i
 					b.Set(slot)
 				}
 			}
-			off += int(run)
+			off += run
 			null = !null
 		}
+		return nil
 	})
 }
 

diff --git a/vng/array.go b/vng/array.go
@@ -58,39 +58,6 @@ func (a *ArrayEncoder) Metadata(off uint64) (uint64, Metadata) {
 	}
 }
 
-type ArrayBuilder struct {
-	Elems   Builder
-	Lengths *Int64Decoder
-}
-
-var _ Builder = (*ArrayBuilder)(nil)
-
-func NewArrayBuilder(array *Array, r io.ReaderAt) (*ArrayBuilder, error) {
-	elems, err := NewBuilder(array.Values, r)
-	if err != nil {
-		return nil, err
-	}
-	return &ArrayBuilder{
-		Elems:   elems,
-		Lengths: NewInt64Decoder(array.Lengths, r),
-	}, nil
-}
-
-func (a *ArrayBuilder) Build(b *zcode.Builder) error {
-	len, err := a.Lengths.Next()
-	if err != nil {
-		return err
-	}
-	b.BeginContainer()
-	for k := 0; k < int(len); k++ {
-		if err := a.Elems.Build(b); err != nil {
-			return err
-		}
-	}
-	b.EndContainer()
-	return nil
-}
-
 type SetEncoder struct {
 	ArrayEncoder
 }

diff --git a/vng/builder.go b/vng/builder.go
diff --git a/vng/dynamic.go b/vng/dynamic.go
@@ -1,11 +1,9 @@
 package vng
 
 import (
-	"fmt"
 	"io"
 
 	"github.com/brimdata/super"
-	"github.com/brimdata/super/zcode"
 	"github.com/brimdata/super/zio"
 	"golang.org/x/sync/errgroup"
 )
@@ -86,84 +84,3 @@ func (d *DynamicEncoder) Emit(w io.Writer) error {
 	}
 	return nil
 }
-
-type dynamicBuilder struct {
-	types   []super.Type
-	tags    *Int64Decoder
-	values  []Builder
-	builder *zcode.Builder
-}
-
-func newDynamicBuilder(zctx *super.Context, d *Dynamic, reader io.ReaderAt) (*dynamicBuilder, error) {
-	values := make([]Builder, 0, len(d.Values))
-	types := make([]super.Type, 0, len(d.Values))
-	for _, val := range d.Values {
-		r, err := NewBuilder(val, reader)
-		if err != nil {
-			return nil, err
-		}
-		values = append(values, r)
-		types = append(types, val.Type(zctx))
-	}
-	return &dynamicBuilder{
-		types:   types,
-		tags:    NewInt64Decoder(d.Tags, reader),
-		values:  values,
-		builder: zcode.NewBuilder(),
-	}, nil
-}
-
-func (d *dynamicBuilder) Read() (*super.Value, error) {
-	b := d.builder
-	b.Truncate()
-	tag, err := d.tags.Next()
-	if err != nil {
-		if err == io.EOF {
-			err = nil
-		}
-		return nil, err
-	}
-	if int(tag) >= len(d.types) {
-		return nil, fmt.Errorf("bad tag encountered scanning VNG dynamic: tag %d when only %d types", tag, len(d.types))
-	}
-	if err := d.values[tag].Build(b); err != nil {
-		return nil, err
-	}
-	return super.NewValue(d.types[tag], b.Bytes().Body()).Ptr(), nil
-}
-
-func NewZedReader(zctx *super.Context, meta Metadata, r io.ReaderAt) (zio.Reader, error) {
-	if d, ok := meta.(*Dynamic); ok {
-		return newDynamicBuilder(zctx, d, r)
-	}
-	values, err := NewBuilder(meta, r)
-	if err != nil {
-		return nil, err
-	}
-	return &vectorBuilder{
-		typ:     meta.Type(zctx),
-		values:  values,
-		builder: zcode.NewBuilder(),
-		count:   meta.Len(),
-	}, nil
-}
-
-type vectorBuilder struct {
-	typ     super.Type
-	values  Builder
-	builder *zcode.Builder
-	count   uint32
-}
-
-func (v *vectorBuilder) Read() (*super.Value, error) {
-	if v.count == 0 {
-		return nil, nil
-	}
-	v.count--
-	b := v.builder
-	b.Truncate()
-	if err := v.values.Build(b); err != nil {
-		return nil, err
-	}
-	return super.NewValue(v.typ, b.Bytes().Body()).Ptr(), nil
-}
diff --git a/vng/field.go b/vng/field.go
@@ -31,21 +31,3 @@ func (f *FieldEncoder) Encode(group *errgroup.Group) {
 func (f *FieldEncoder) Emit(w io.Writer) error {
 	return f.values.Emit(w)
 }
-
-type FieldBuilder struct {
-	Values Builder
-}
-
-func NewFieldBuilder(field Field, r io.ReaderAt) (*FieldBuilder, error) {
-	values, err := NewBuilder(field.Values, r)
-	if err != nil {
-		return nil, err
-	}
-	return &FieldBuilder{
-		Values: values,
-	}, nil
-}
-
-func (f *FieldBuilder) Build(b *zcode.Builder) error {
-	return f.Values.Build(b)
-}
diff --git a/vng/int.go b/vng/int.go
@@ -4,6 +4,7 @@ import (
 	"io"
 
 	"github.com/brimdata/super"
+	"github.com/brimdata/super/zcode"
 )
 
 type Int64Encoder struct {
@@ -18,18 +19,17 @@ func (p *Int64Encoder) Write(v int64) {
 	p.PrimitiveEncoder.Write(super.EncodeInt(v))
 }
 
-type Int64Decoder struct {
-	PrimitiveBuilder
-}
-
-func NewInt64Decoder(loc Segment, r io.ReaderAt) *Int64Decoder {
-	return &Int64Decoder{*NewPrimitiveBuilder(&Primitive{Typ: super.TypeInt64, Location: loc}, r)}
-}
-
-func (p *Int64Decoder) Next() (int64, error) {
-	zv, err := p.ReadBytes()
-	if err != nil {
-		return 0, err
+func ReadUint32s(loc Segment, r io.ReaderAt) ([]uint32, error) {
+	buf := make([]byte, loc.MemLength)
+	if err := loc.Read(r, buf); err != nil {
+		if err == io.EOF {
+			return nil, nil
+		}
+		return nil, err
+	}
+	var vals []uint32
+	for it := zcode.Iter(buf); !it.Done(); {
+		vals = append(vals, uint32(super.DecodeInt(it.Next())))
 	}
-	return super.DecodeInt(zv), err
+	return vals, nil
 }
diff --git a/vng/map.go b/vng/map.go
@@ -62,45 +62,3 @@ func (m *MapEncoder) Encode(group *errgroup.Group) {
 	m.keys.Encode(group)
 	m.values.Encode(group)
 }
-
-type MapBuilder struct {
-	Keys    Builder
-	Values  Builder
-	Lengths *Int64Decoder
-}
-
-var _ Builder = (*MapBuilder)(nil)
-
-func NewMapBuilder(m *Map, r io.ReaderAt) (*MapBuilder, error) {
-	keys, err := NewBuilder(m.Keys, r)
-	if err != nil {
-		return nil, err
-	}
-	values, err := NewBuilder(m.Values, r)
-	if err != nil {
-		return nil, err
-	}
-	return &MapBuilder{
-		Keys:    keys,
-		Values:  values,
-		Lengths: NewInt64Decoder(m.Lengths, r),
-	}, nil
-}
-
-func (m *MapBuilder) Build(b *zcode.Builder) error {
-	len, err := m.Lengths.Next()
-	if err != nil {
-		return err
-	}
-	b.BeginContainer()
-	for k := 0; k < int(len); k++ {
-		if err := m.Keys.Build(b); err != nil {
-			return err
-		}
-		if err := m.Values.Build(b); err != nil {
-			return err
-		}
-	}
-	b.EndContainer()
-	return nil
-}
diff --git a/vng/nulls.go b/vng/nulls.go
@@ -87,40 +87,3 @@ func (n *NullsEncoder) Emit(w io.Writer) error {
 	}
 	return nil
 }
-
-type NullsBuilder struct {
-	Values Builder
-	Runs   Int64Decoder
-	null   bool
-	run    int
-}
-
-var _ (Builder) = (*NullsBuilder)(nil)
-
-func NewNullsBuilder(values Builder, loc Segment, r io.ReaderAt) *NullsBuilder {
-	// We start out with null true so it is immediately flipped to
-	// false on the first call to Read.
-	return &NullsBuilder{
-		Values: values,
-		Runs:   *NewInt64Decoder(loc, r),
-		null:   true,
-	}
-}
-
-func (n *NullsBuilder) Build(b *zcode.Builder) error {
-	run := n.run
-	for run == 0 {
-		n.null = !n.null
-		v, err := n.Runs.Next()
-		if err != nil {
-			return err
-		}
-		run = int(v)
-	}
-	n.run = run - 1
-	if n.null {
-		b.Append(nil)
-		return nil
-	}
-	return n.Values.Build(b)
-}