refactor "super dev compile" into "super compile" and "super db compile"

cli/lakeflags/flags.go

@@ -75,7 +75,7 @@ func (l *Flags) Open(ctx context.Context) (api.Interface, error) {
 	}
 	lk, err := api.OpenLocalLake(ctx, zap.Must(zap.NewProduction()), uri.String())
 	if errors.Is(err, lake.ErrNotExist) {
-		return nil, fmt.Errorf("%w\n(hint: run 'zed init' to initialize lake at this location)", err)
+		return nil, fmt.Errorf("%w\n(hint: run 'super db init' to initialize lake at this location)", err)
 	}
 	return lk, err
 }

cmd/super/compile/command.go

@@ -0,0 +1,59 @@
+package compile
+
+import (
+	"flag"
+
+	"github.com/brimdata/super/cmd/super/root"
+	"github.com/brimdata/super/pkg/charm"
+)
+
+var spec = &charm.Spec{
+	Name:  "compile",
+	Usage: "compile [ options ] spq|sql",
+	Short: "compile a local query for inspection and debugging",
+	Long: `
+This command parses a query and emits the resulting abstract syntax
+tree (AST) or runtime directed-acycle graph (DAG) in the output format desired.
+Use "-dag" to specify the DAG form; otherwise, the AST form is assumed.
+
+The query text may be either SQL or SPQ.  To force parsing as SQL,
+use the "-sql" flag.
+
+The "-C" option causes the output to be shown as query language source
+instead of the AST.  This is particularly helpful to see how SQP queries
+in their abbreviated form are translated into the exanded, pedantic form 
+of piped SQL.  The DAG can also be formatted as query-style text
+but the resulting text is informational only and does not conform to
+any query syntax.  When "-C" is specified, the result is sent to stdout
+and the "-f" and "-o" options have no effect.
+
+This command is often used for dev and test but
+is also useful to advanced users for understanding how SQL and SPQ syntax is
+parsed into an AST or compiled into a runtime DAG.
+`,
+	New: New,
+}
+
+func init() {
+	root.Super.Add(spec)
+}
+
+type Command struct {
+	*root.Command
+	shared Shared
+}
+
+func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
+	c := &Command{Command: parent.(*root.Command)}
+	c.shared.SetFlags(f)
+	return c, nil
+}
+
+func (c *Command) Run(args []string) error {
+	ctx, cleanup, err := c.Init(&c.shared.OutputFlags)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+	return c.shared.Run(ctx, args, nil, false)
+}

cmd/super/compile/shared.go

@@ -0,0 +1,135 @@
+package compile
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+
+	"github.com/brimdata/super"
+	"github.com/brimdata/super/cli/lakeflags"
+	"github.com/brimdata/super/cli/outputflags"
+	"github.com/brimdata/super/cli/queryflags"
+	"github.com/brimdata/super/compiler"
+	"github.com/brimdata/super/compiler/data"
+	"github.com/brimdata/super/compiler/describe"
+	"github.com/brimdata/super/compiler/parser"
+	"github.com/brimdata/super/lake"
+	"github.com/brimdata/super/pkg/storage"
+	"github.com/brimdata/super/runtime"
+	"github.com/brimdata/super/zfmt"
+	"github.com/brimdata/super/zio"
+	"github.com/brimdata/super/zson"
+)
+
+type Shared struct {
+	dag         bool
+	includes    queryflags.Includes
+	optimize    bool
+	parallel    int
+	query       bool
+	sql         bool
+	OutputFlags outputflags.Flags
+}
+
+func (s *Shared) SetFlags(fs *flag.FlagSet) {
+	fs.BoolVar(&s.dag, "dag", false, "display output as DAG (implied by -O or -P)")
+	fs.Var(&s.includes, "I", "source file containing query text (may be repeated)")
+	fs.BoolVar(&s.optimize, "O", false, "display optimized DAG")
+	fs.IntVar(&s.parallel, "P", 0, "display parallelized DAG")
+	fs.BoolVar(&s.query, "C", false, "display DAG or AST as query text")
+	fs.BoolVar(&s.sql, "sql", false, "force a strict SQL intepretation of the query text")
+	s.OutputFlags.SetFlags(fs)
+}
+
+func (s *Shared) Run(ctx context.Context, args []string, lakeFlags *lakeflags.Flags, desc bool) error {
+	if len(s.includes) == 0 && len(args) == 0 {
+		return errors.New("no query specified")
+	}
+	if len(args) > 1 {
+		return errors.New("too many arguments")
+	}
+	var lk *lake.Root
+	if lakeFlags != nil {
+		lakeAPI, err := lakeFlags.Open(ctx)
+		if err != nil {
+			return err
+		}
+		lk = lakeAPI.Root()
+	}
+	var query string
+	if len(args) == 1 {
+		query = args[0]
+	}
+	seq, sset, err := compiler.Parse(query, s.includes...)
+	if err != nil {
+		return err
+	}
+	if s.optimize || s.parallel > 0 || desc {
+		s.dag = true
+	}
+	if !s.dag {
+		if s.query {
+			fmt.Println(zfmt.AST(seq))
+			return nil
+		}
+		return s.writeValue(ctx, seq)
+	}
+	runtime, err := compiler.NewJob(runtime.DefaultContext(), seq, data.NewSource(nil, lk), nil)
+	if err != nil {
+		if list, ok := err.(parser.ErrorList); ok {
+			list.SetSourceSet(sset)
+		}
+		return err
+	}
+	if desc {
+		description, err := describe.AnalyzeDAG(ctx, runtime.Entry(), data.NewSource(nil, lk), nil)
+		if err != nil {
+			return err
+		}
+		return s.writeValue(ctx, description)
+	}
+	if s.parallel > 0 {
+		if err := runtime.Optimize(); err != nil {
+			return err
+		}
+		if err := runtime.Parallelize(s.parallel); err != nil {
+			return err
+		}
+	} else if s.optimize {
+		if err := runtime.Optimize(); err != nil {
+			return err
+		}
+	}
+	if s.query {
+		fmt.Println(zfmt.DAG(runtime.Entry()))
+		return nil
+	}
+	return s.writeValue(ctx, runtime.Entry())
+}
+
+func (s *Shared) writeValue(ctx context.Context, v any) error {
+	val, err := zson.MarshalZNG(v)
+	if err != nil {
+		return err
+	}
+	writer, err := s.OutputFlags.Open(ctx, storage.NewLocalEngine())
+	if err != nil {
+		return err
+	}
+	err = zio.CopyWithContext(ctx, writer, &valReader{&val})
+	if closeErr := writer.Close(); err == nil {
+		err = closeErr
+	}
+	return err
+}
+
+type valReader struct {
+	*super.Value
+}
+
+func (r *valReader) Read() (*super.Value, error) {
+	val := r.Value
+	r.Value = nil
+	return val, nil
+}

cmd/super/db/auth/command.go

@@ -33,5 +33,5 @@ func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
 }
 
 func (c *Command) Run(args []string) error {
-	return charm.ErrNoRun
+	return charm.NoRun(args)
 }

cmd/super/db/command.go

@@ -34,14 +34,5 @@ func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
 }
 
 func (c *Command) Run(args []string) error {
-	//XXX
-	_, cancel, err := c.Init()
-	if err != nil {
-		return err
-	}
-	defer cancel()
-	if len(args) == 0 {
-		return charm.NeedHelp
-	}
-	return charm.ErrNoRun
+	return charm.NoRun(args)
 }

cmd/super/db/compile/command.go

@@ -0,0 +1,51 @@
+package compile
+
+import (
+	"flag"
+
+	"github.com/brimdata/super/cmd/super/compile"
+	"github.com/brimdata/super/cmd/super/db"
+	"github.com/brimdata/super/pkg/charm"
+)
+
+var spec = &charm.Spec{
+	Name:  "compile",
+	Usage: "compile [ options ] spq|sql",
+	Short: "compile a lake query for inspection and debugging",
+	Long: `
+The "super db compile" command is just like the "super compile" command except 
+it compiles the query for a SuperDB data lake instead of a local file system.
+The primary difference here is that "from" operators on a lake work with data
+stored in the lake whereas "from" operators on file system work with local files.
+In both cases, "from" can also retrieve data from HTTP APIs via URL.
+
+See the "super compile" command help for futher information.
+`,
+	New: New,
+}
+
+func init() {
+	db.Spec.Add(spec)
+}
+
+type Command struct {
+	parent   *db.Command
+	shared   compile.Shared
+	describe bool
+}
+
+func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
+	c := &Command{parent: parent.(*db.Command)}
+	c.shared.SetFlags(f)
+	f.BoolVar(&c.describe, "describe", false, "emit describe endpoint results for this query")
+	return c, nil
+}
+
+func (c *Command) Run(args []string) error {
+	ctx, cleanup, err := c.parent.Init(&c.shared.OutputFlags)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+	return c.shared.Run(ctx, args, &c.parent.LakeFlags, c.describe)
+}

cmd/super/db/vector/command.go

@@ -33,8 +33,5 @@ func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
 }
 
 func (c *Command) Run(args []string) error {
-	if len(args) == 0 {
-		return charm.NeedHelp
-	}
-	return charm.ErrNoRun
+	return charm.NoRun(args)
 }

cmd/super/db/ztests/init-hint.yaml

@@ -1,10 +1,10 @@
 script: |
-  ! super db -lake=path/to/zed ls
+  ! super db -lake=path/to/superdb ls
 
 outputs:
   - name: stderr
     regexp: |
       .* lake does not exist
-      \(hint: run 'zed init' to initialize lake at this location\)
+      \(hint: run 'super db init' to initialize lake at this location\)
   - name: stdout
     data: ""

cmd/super/dev/command.go

@@ -17,10 +17,18 @@ prints the list of known dev tools.`,
 	New: New,
 }
 
+type Command struct {
+	*root.Command
+}
+
 func init() {
 	root.Super.Add(Spec)
 }
 
 func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
-	return parent.(*root.Command), nil
+	return &Command{Command: parent.(*root.Command)}, nil
+}
+
+func (c *Command) Run(args []string) error {
+	return charm.NoRun(args)
 }