simplify seek index writer and CLI arg (#2742)

This commit simplifies the seek index writer by moving the
logic for auto-stream-termination out of zngio and into
segment.Writer.  We also deleted the unused indexing code
in zngio along with the "zed index seek" command and related
support.

As previously agreed among the team, the streamax argument
to zed lake add and zed lake load was changed to seekstride
and the units changed from records to bytes.

Closes #2542

cli/outputflags/flags.go

@@ -37,7 +37,6 @@ func (f *Flags) setFlags(fs *flag.FlagSet) {
 	fs.BoolVar(&f.Text.ShowTypes, "T", false, "display field types in text output")
 	fs.BoolVar(&f.Text.ShowFields, "F", false, "display field names in text output")
 	fs.BoolVar(&f.color, "color", true, "enable/disable color formatting for -Z and lake text output")
-	fs.IntVar(&f.Zng.StreamRecordsMax, "b", 0, "limit for number of records in each ZNG stream (0 for no limit)")
 	fs.IntVar(&f.Zng.LZ4BlockSize, "znglz4blocksize", zngio.DefaultLZ4BlockSize,
 		"LZ4 block size in bytes for ZNG compression (nonpositive to disable)")
 	fs.IntVar(&f.ZSON.Pretty, "pretty", 4,

cmd/zed/lake/add/command.go

@@ -11,6 +11,7 @@ import (
 	"github.com/brimdata/zed/pkg/charm"
 	"github.com/brimdata/zed/pkg/rlimit"
 	"github.com/brimdata/zed/pkg/storage"
+	"github.com/brimdata/zed/pkg/units"
 	"github.com/brimdata/zed/zbuf"
 	"github.com/brimdata/zed/zio"
 	"github.com/brimdata/zed/zson"
@@ -45,17 +46,20 @@ func init() {
 // TBD: add option to apply Zed program on add path?
 
 type Command struct {
-	lake                  *zedlake.Command
-	importStreamRecordMax int
-	commit                bool
-	inputFlags            inputflags.Flags
+	lake       *zedlake.Command
+	seekStride units.Bytes
+	commit     bool
+	inputFlags inputflags.Flags
 	zedlake.CommitFlags
 }
 
 func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
-	c := &Command{lake: parent.(*zedlake.Command)}
+	c := &Command{
+		lake:       parent.(*zedlake.Command),
+		seekStride: units.Bytes(lake.SeekIndexStride),
+	}
 	f.BoolVar(&c.commit, "commit", false, "commit added data if successfully written")
-	f.IntVar(&c.importStreamRecordMax, "streammax", lake.ImportStreamRecordsMax, "limit for number of records in each ZNG stream (0 for no limit)")
+	f.Var(&c.seekStride, "seekstride", "size of seek-index unit for ZNG data, as '32KB', '1MB', etc.")
 	c.inputFlags.SetFlags(f)
 	c.CommitFlags.SetFlags(f)
 	return c, nil
@@ -70,7 +74,7 @@ func (c *Command) Run(args []string) error {
 	if len(args) == 0 {
 		return errors.New("zed lake add: at least one input file must be specified (- for stdin)")
 	}
-	lake.ImportStreamRecordsMax = c.importStreamRecordMax
+	lake.SeekIndexStride = int(c.seekStride)
 	if _, err := rlimit.RaiseOpenFilesLimit(); err != nil {
 		return err
 	}

cmd/zed/lake/load/command.go

@@ -12,6 +12,7 @@ import (
 	"github.com/brimdata/zed/pkg/charm"
 	"github.com/brimdata/zed/pkg/rlimit"
 	"github.com/brimdata/zed/pkg/storage"
+	"github.com/brimdata/zed/pkg/units"
 	"github.com/brimdata/zed/zbuf"
 	"github.com/brimdata/zed/zio"
 	"github.com/brimdata/zed/zson"
@@ -33,17 +34,20 @@ func init() {
 }
 
 type Command struct {
-	lake                  *zedlake.Command
-	importStreamRecordMax int
-	commit                bool
+	lake       *zedlake.Command
+	seekStride units.Bytes
+	commit     bool
 	zedlake.CommitFlags
 	procFlags  procflags.Flags
 	inputFlags inputflags.Flags
 }
 
 func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
-	c := &Command{lake: parent.(*zedlake.Command)}
-	f.IntVar(&c.importStreamRecordMax, "streammax", lake.ImportStreamRecordsMax, "limit for number of records in each ZNG stream (0 for no limit)")
+	c := &Command{
+		lake:       parent.(*zedlake.Command),
+		seekStride: units.Bytes(lake.SeekIndexStride),
+	}
+	f.Var(&c.seekStride, "seekstride", "size of seek-index unit for ZNG data, as '32KB', '1MB', etc.")
 	c.CommitFlags.SetFlags(f)
 	c.inputFlags.SetFlags(f)
 	c.procFlags.SetFlags(f)
@@ -59,7 +63,7 @@ func (c *Command) Run(args []string) error {
 	if len(args) == 0 {
 		return errors.New("zed lake load: at least one input file must be specified (- for stdin)")
 	}
-	lake.ImportStreamRecordsMax = c.importStreamRecordMax
+	lake.SeekIndexStride = int(c.seekStride)
 	if _, err := rlimit.RaiseOpenFilesLimit(); err != nil {
 		return err
 	}

cmd/zed/main.go

@@ -20,7 +20,6 @@ import (
 	_ "github.com/brimdata/zed/cmd/zed/index/create"
 	_ "github.com/brimdata/zed/cmd/zed/index/lookup"
 	_ "github.com/brimdata/zed/cmd/zed/index/section"
-	_ "github.com/brimdata/zed/cmd/zed/index/seek"
 	"github.com/brimdata/zed/cmd/zed/lake"
 	_ "github.com/brimdata/zed/cmd/zed/lake/add"
 	_ "github.com/brimdata/zed/cmd/zed/lake/commit"