refactor "super dev compile" into "super compile" and "super db compi…

…le" (#5366)

This commit reparents the "super dev compile" command under the top-level
"super" command and creates a parallel command "super db compile".
This makes it much clearer as to when you are compiling a query for
a lake vs a file system.

We also changed the compile commands to take a single query argument
instead of concatenating multiple arguments.  Also, the command options
are a little different: you now say "-dag" if you want the DAG output
and get the AST output otherwise.

There were some inconsitencies in how commands were parented so
we added charm.NoRun() and fixed up the command hierarchy linkages.
The invariant should now be that every internal command properly appears
in the command hierarchy and NoRun() is invoked by the Run() method
of an interior command to give proper help.

The command-line help has also been updated.

cli/lakeflags/flags.go

@@ -75,7 +75,7 @@ func (l *Flags) Open(ctx context.Context) (api.Interface, error) {
 	}
 	lk, err := api.OpenLocalLake(ctx, zap.Must(zap.NewProduction()), uri.String())
 	if errors.Is(err, lake.ErrNotExist) {
-		return nil, fmt.Errorf("%w\n(hint: run 'zed init' to initialize lake at this location)", err)
+		return nil, fmt.Errorf("%w\n(hint: run 'super db init' to initialize lake at this location)", err)
 	}
 	return lk, err
 }

cmd/super/compile/command.go

@@ -0,0 +1,59 @@
+package compile
+
+import (
+	"flag"
+
+	"github.com/brimdata/super/cmd/super/root"
+	"github.com/brimdata/super/pkg/charm"
+)
+
+var spec = &charm.Spec{
+	Name:  "compile",
+	Usage: "compile [ options ] spq|sql",
+	Short: "compile a local query for inspection and debugging",
+	Long: `
+This command parses a query and emits the resulting abstract syntax
+tree (AST) or runtime directed acyclic graph (DAG) in the output format desired.
+Use "-dag" to specify the DAG form; otherwise, the AST form is assumed.
+
+The query text may be either SQL or SPQ.  To force parsing as SQL,
+use the "-sql" flag.
+
+The "-C" option causes the output to be shown as query language source
+instead of the AST.  This is particularly helpful to see how SQP queries
+in their abbreviated form are translated into the exanded, pedantic form 
+of piped SQL.  The DAG can also be formatted as query-style text
+but the resulting text is informational only and does not conform to
+any query syntax.  When "-C" is specified, the result is sent to stdout
+and the "-f" and "-o" options have no effect.
+
+This command is often used for dev and test but
+is also useful to advanced users for understanding how SQL and SPQ syntax is
+parsed into an AST or compiled into a runtime DAG.
+`,
+	New: New,
+}
+
+func init() {
+	root.Super.Add(spec)
+}
+
+type Command struct {
+	*root.Command
+	shared Shared
+}
+
+func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
+	c := &Command{Command: parent.(*root.Command)}
+	c.shared.SetFlags(f)
+	return c, nil
+}
+
+func (c *Command) Run(args []string) error {
+	ctx, cleanup, err := c.Init(&c.shared.OutputFlags)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+	return c.shared.Run(ctx, args, nil, false)
+}

cmd/super/compile/shared.go

@@ -0,0 +1,126 @@
+package compile
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+
+	"github.com/brimdata/super"
+	"github.com/brimdata/super/cli/lakeflags"
+	"github.com/brimdata/super/cli/outputflags"
+	"github.com/brimdata/super/cli/queryflags"
+	"github.com/brimdata/super/compiler"
+	"github.com/brimdata/super/compiler/data"
+	"github.com/brimdata/super/compiler/describe"
+	"github.com/brimdata/super/compiler/parser"
+	"github.com/brimdata/super/lake"
+	"github.com/brimdata/super/pkg/storage"
+	"github.com/brimdata/super/runtime"
+	"github.com/brimdata/super/zbuf"
+	"github.com/brimdata/super/zfmt"
+	"github.com/brimdata/super/zio"
+	"github.com/brimdata/super/zson"
+)
+
+type Shared struct {
+	dag         bool
+	includes    queryflags.Includes
+	optimize    bool
+	parallel    int
+	query       bool
+	sql         bool
+	OutputFlags outputflags.Flags
+}
+
+func (s *Shared) SetFlags(fs *flag.FlagSet) {
+	fs.BoolVar(&s.dag, "dag", false, "display output as DAG (implied by -O or -P)")
+	fs.Var(&s.includes, "I", "source file containing query text (may be repeated)")
+	fs.BoolVar(&s.optimize, "O", false, "display optimized DAG")
+	fs.IntVar(&s.parallel, "P", 0, "display parallelized DAG")
+	fs.BoolVar(&s.query, "C", false, "display DAG or AST as query text")
+	fs.BoolVar(&s.sql, "sql", false, "force a strict SQL intepretation of the query text")
+	s.OutputFlags.SetFlags(fs)
+}
+
+func (s *Shared) Run(ctx context.Context, args []string, lakeFlags *lakeflags.Flags, desc bool) error {
+	if len(s.includes) == 0 && len(args) == 0 {
+		return errors.New("no query specified")
+	}
+	if len(args) > 1 {
+		return errors.New("too many arguments")
+	}
+	var lk *lake.Root
+	if lakeFlags != nil {
+		lakeAPI, err := lakeFlags.Open(ctx)
+		if err != nil {
+			return err
+		}
+		lk = lakeAPI.Root()
+	}
+	var query string
+	if len(args) == 1 {
+		query = args[0]
+	}
+	seq, sset, err := compiler.Parse(query, s.includes...)
+	if err != nil {
+		return err
+	}
+	if s.optimize || s.parallel > 0 || desc {
+		s.dag = true
+	}
+	if !s.dag {
+		if s.query {
+			fmt.Println(zfmt.AST(seq))
+			return nil
+		}
+		return s.writeValue(ctx, seq)
+	}
+	runtime, err := compiler.NewJob(runtime.DefaultContext(), seq, data.NewSource(nil, lk), nil)
+	if err != nil {
+		if list, ok := err.(parser.ErrorList); ok {
+			list.SetSourceSet(sset)
+		}
+		return err
+	}
+	if desc {
+		description, err := describe.AnalyzeDAG(ctx, runtime.Entry(), data.NewSource(nil, lk), nil)
+		if err != nil {
+			return err
+		}
+		return s.writeValue(ctx, description)
+	}
+	if s.parallel > 0 {
+		if err := runtime.Optimize(); err != nil {
+			return err
+		}
+		if err := runtime.Parallelize(s.parallel); err != nil {
+			return err
+		}
+	} else if s.optimize {
+		if err := runtime.Optimize(); err != nil {
+			return err
+		}
+	}
+	if s.query {
+		fmt.Println(zfmt.DAG(runtime.Entry()))
+		return nil
+	}
+	return s.writeValue(ctx, runtime.Entry())
+}
+
+func (s *Shared) writeValue(ctx context.Context, v any) error {
+	val, err := zson.MarshalZNG(v)
+	if err != nil {
+		return err
+	}
+	writer, err := s.OutputFlags.Open(ctx, storage.NewLocalEngine())
+	if err != nil {
+		return err
+	}
+	err = zio.CopyWithContext(ctx, writer, zbuf.NewArray([]super.Value{val}))
+	if closeErr := writer.Close(); err == nil {
+		err = closeErr
+	}
+	return err
+}

cmd/super/db/auth/command.go

@@ -33,5 +33,5 @@ func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
 }
 
 func (c *Command) Run(args []string) error {
-	return charm.ErrNoRun
+	return charm.NoRun(args)
 }

cmd/super/db/command.go

@@ -34,14 +34,5 @@ func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
 }
 
 func (c *Command) Run(args []string) error {
-	//XXX
-	_, cancel, err := c.Init()
-	if err != nil {
-		return err
-	}
-	defer cancel()
-	if len(args) == 0 {
-		return charm.NeedHelp
-	}
-	return charm.ErrNoRun
+	return charm.NoRun(args)
 }

cmd/super/db/compile/command.go

@@ -0,0 +1,51 @@
+package compile
+
+import (
+	"flag"
+
+	"github.com/brimdata/super/cmd/super/compile"
+	"github.com/brimdata/super/cmd/super/db"
+	"github.com/brimdata/super/pkg/charm"
+)
+
+var spec = &charm.Spec{
+	Name:  "compile",
+	Usage: "compile [ options ] spq|sql",
+	Short: "compile a lake query for inspection and debugging",
+	Long: `
+The "super db compile" command is just like the "super compile" command except 
+it compiles the query for a SuperDB data lake instead of a local file system.
+The primary difference here is that "from" operators on a lake work with data
+stored in the lake whereas "from" operators on file system work with local files.
+In both cases, "from" can also retrieve data from HTTP APIs via URL.
+
+See the "super compile" command help for futher information.
+`,
+	New: New,
+}
+
+func init() {
+	db.Spec.Add(spec)
+}
+
+type Command struct {
+	parent   *db.Command
+	shared   compile.Shared
+	describe bool
+}
+
+func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
+	c := &Command{parent: parent.(*db.Command)}
+	c.shared.SetFlags(f)
+	f.BoolVar(&c.describe, "describe", false, "emit describe endpoint results for this query")
+	return c, nil
+}
+
+func (c *Command) Run(args []string) error {
+	ctx, cleanup, err := c.parent.Init(&c.shared.OutputFlags)
+	if err != nil {
+		return err
+	}
+	defer cleanup()
+	return c.shared.Run(ctx, args, &c.parent.LakeFlags, c.describe)
+}

cmd/super/db/vector/command.go

@@ -33,8 +33,5 @@ func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
 }
 
 func (c *Command) Run(args []string) error {
-	if len(args) == 0 {
-		return charm.NeedHelp
-	}
-	return charm.ErrNoRun
+	return charm.NoRun(args)
 }

cmd/super/db/ztests/init-hint.yaml

@@ -1,10 +1,10 @@
 script: |
-  ! super db -lake=path/to/zed ls
+  ! super db -lake=path/to/superdb ls
 
 outputs:
   - name: stderr
     regexp: |
       .* lake does not exist
-      \(hint: run 'zed init' to initialize lake at this location\)
+      \(hint: run 'super db init' to initialize lake at this location\)
   - name: stdout
     data: ""

cmd/super/dev/command.go

@@ -17,10 +17,18 @@ prints the list of known dev tools.`,
 	New: New,
 }
 
+type Command struct {
+	*root.Command
+}
+
 func init() {
 	root.Super.Add(Spec)
 }
 
 func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
-	return parent.(*root.Command), nil
+	return &Command{Command: parent.(*root.Command)}, nil
+}
+
+func (c *Command) Run(args []string) error {
+	return charm.NoRun(args)
 }