diff --git a/api/api.go b/api/api.go index 4e64653da0..76343cf7f3 100644 --- a/api/api.go +++ b/api/api.go @@ -4,7 +4,6 @@ import ( "context" "github.com/brimdata/super/compiler/srcfiles" - "github.com/brimdata/super/lakeparse" "github.com/brimdata/super/order" "github.com/brimdata/super/pkg/field" "github.com/brimdata/super/pkg/nano" @@ -98,8 +97,7 @@ type EventBranch struct { } type QueryRequest struct { - Query string `json:"query"` - Head lakeparse.Commitish `json:"head"` + Query string `json:"query"` } type QueryChannelSet struct { diff --git a/api/client/connection.go b/api/client/connection.go index 0a0db7bfec..30285d16a5 100644 --- a/api/client/connection.go +++ b/api/client/connection.go @@ -19,7 +19,6 @@ import ( "github.com/brimdata/super/compiler/srcfiles" "github.com/brimdata/super/lake" "github.com/brimdata/super/lake/branches" - "github.com/brimdata/super/lakeparse" "github.com/brimdata/super/runtime/exec" "github.com/brimdata/super/zio/zngio" "github.com/brimdata/super/zson" @@ -296,15 +295,12 @@ func (c *Connection) Revert(ctx context.Context, poolID ksuid.KSUID, branchName // // As for Connection.Do, if the returned error is nil, the user is expected to // call Response.Body.Close. -func (c *Connection) Query(ctx context.Context, head *lakeparse.Commitish, src string, filenames ...string) (*Response, error) { +func (c *Connection) Query(ctx context.Context, src string, filenames ...string) (*Response, error) { files, err := srcfiles.Concat(filenames, src) if err != nil { return nil, err } body := api.QueryRequest{Query: string(files.Text)} - if head != nil { - body.Head = *head - } req := c.NewRequest(ctx, http.MethodPost, "/query?ctrl=T", body) res, err := c.Do(req) if ae := (*api.Error)(nil); errors.As(err, &ae) && len(ae.CompilationErrors) > 0 { diff --git a/cmd/super/compile/shared.go b/cmd/super/compile/shared.go index 75113d7745..aefec27259 100644 --- a/cmd/super/compile/shared.go +++ b/cmd/super/compile/shared.go @@ -76,12 +76,12 @@ func (s *Shared) Run(ctx context.Context, args []string, lakeFlags *lakeflags.Fl } return s.writeValue(ctx, ast.Parsed()) } - runtime, err := compiler.NewJob(runtime.DefaultContext(), ast, data.NewSource(nil, lk), nil) + runtime, err := compiler.NewJob(runtime.DefaultContext(), ast, data.NewSource(nil, lk), false) if err != nil { return err } if desc { - description, err := describe.AnalyzeDAG(ctx, runtime.Entry(), data.NewSource(nil, lk), nil) + description, err := describe.AnalyzeDAG(ctx, runtime.Entry(), data.NewSource(nil, lk)) if err != nil { return err } diff --git a/cmd/super/db/branch/command.go b/cmd/super/db/branch/command.go index 6d3c70f9df..2766c4a87a 100644 --- a/cmd/super/db/branch/command.go +++ b/cmd/super/db/branch/command.go @@ -131,7 +131,7 @@ func (c *Command) list(ctx context.Context, lake api.Interface) error { if err != nil { return err } - q, err := lake.Query(ctx, nil, query) + q, err := lake.Query(ctx, query) if err != nil { w.Close() return err diff --git a/cmd/super/db/log/command.go b/cmd/super/db/log/command.go index 5536483756..de2247884b 100644 --- a/cmd/super/db/log/command.go +++ b/cmd/super/db/log/command.go @@ -72,7 +72,7 @@ func (c *Command) Run(args []string) error { return err } defer w.Close() - q, err := lake.Query(ctx, nil, query) + q, err := lake.Query(ctx, query) if err != nil { return err } diff --git a/cmd/super/db/ls/command.go b/cmd/super/db/ls/command.go index bd41be1272..82d1723137 100644 --- a/cmd/super/db/ls/command.go +++ b/cmd/super/db/ls/command.go @@ -88,7 +88,7 @@ func (c *Command) Run(args []string) error { if err != nil { return err } - q, err := lake.Query(ctx, nil, query) + q, err := lake.Query(ctx, query) if err != nil { w.Close() return err diff --git a/cmd/super/db/query/command.go b/cmd/super/db/query/command.go index 2e694e7a44..79343b881e 100644 --- a/cmd/super/db/query/command.go +++ b/cmd/super/db/query/command.go @@ -5,7 +5,6 @@ import ( "os" "github.com/brimdata/super/cli/outputflags" - "github.com/brimdata/super/cli/poolflags" "github.com/brimdata/super/cli/queryflags" "github.com/brimdata/super/cli/runtimeflags" "github.com/brimdata/super/cmd/super/db" @@ -33,7 +32,6 @@ func init() { type Command struct { *db.Command outputFlags outputflags.Flags - poolFlags poolflags.Flags queryFlags queryflags.Flags runtimeFlags runtimeflags.Flags } @@ -41,7 +39,6 @@ type Command struct { func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) { c := &Command{Command: parent.(*db.Command)} c.outputFlags.SetFlags(f) - c.poolFlags.SetFlags(f) c.queryFlags.SetFlags(f) c.runtimeFlags.SetFlags(f) return c, nil @@ -68,8 +65,7 @@ func (c *Command) Run(args []string) error { if err != nil { return err } - head, _ := c.poolFlags.HEAD() - query, err := lake.Query(ctx, head, src, c.queryFlags.Includes...) + query, err := lake.Query(ctx, src, c.queryFlags.Includes...) if err != nil { w.Close() return err diff --git a/cmd/super/internal/lakemanage/scan.go b/cmd/super/internal/lakemanage/scan.go index e4ccb87bed..14cfd1f745 100644 --- a/cmd/super/internal/lakemanage/scan.go +++ b/cmd/super/internal/lakemanage/scan.go @@ -74,7 +74,7 @@ type objectIterator struct { func newObjectIterator(ctx context.Context, lake api.Interface, head *lakeparse.Commitish) (*objectIterator, error) { query := fmt.Sprintf(iteratorQuery, head.Pool, head.Branch, head.Pool, head.Branch) - q, err := lake.Query(ctx, nil, query) + q, err := lake.Query(ctx, query) if err != nil { return nil, err } diff --git a/compiler/ast/ast.go b/compiler/ast/ast.go index 34a4f2fbe0..ad2bbab0d0 100644 --- a/compiler/ast/ast.go +++ b/compiler/ast/ast.go @@ -574,8 +574,10 @@ type ( Loc `json:"loc"` } Delete struct { - Kind string `json:"kind" unpack:""` - Loc `json:"loc"` // dummy field, not needed except to implement Node + Kind string `json:"kind" unpack:""` + Pool string `json:"pool"` + Branch string `json:"branch"` + Loc `json:"loc"` // dummy field, not needed except to implement Node } ) diff --git a/compiler/dag/op.go b/compiler/dag/op.go index 7d31bdb685..623d6ac302 100644 --- a/compiler/dag/op.go +++ b/compiler/dag/op.go @@ -246,6 +246,9 @@ type ( Kind string `json:"kind" unpack:""` Meta string `json:"meta"` } + NullScan struct { + Kind string `json:"kind" unpack:""` + } ) var LakeMetas = map[string]struct{}{ @@ -273,6 +276,7 @@ func (*DeleteScan) OpNode() {} func (*LakeMetaScan) OpNode() {} func (*PoolMetaScan) OpNode() {} func (*CommitMetaScan) OpNode() {} +func (*NullScan) OpNode() {} func (*Lister) OpNode() {} func (*Slicer) OpNode() {} diff --git a/compiler/dag/unpack.go b/compiler/dag/unpack.go index 76d105da6f..572e49fcb1 100644 --- a/compiler/dag/unpack.go +++ b/compiler/dag/unpack.go @@ -42,6 +42,7 @@ var unpacker = unpack.New( MapExpr{}, Merge{}, Mirror{}, + NullScan{}, Output{}, Over{}, OverExpr{}, diff --git a/compiler/describe/analyze.go b/compiler/describe/analyze.go index 61e65dec08..8845e11997 100644 --- a/compiler/describe/analyze.go +++ b/compiler/describe/analyze.go @@ -10,7 +10,6 @@ import ( "github.com/brimdata/super/compiler/parser" "github.com/brimdata/super/compiler/semantic" "github.com/brimdata/super/lake" - "github.com/brimdata/super/lakeparse" "github.com/brimdata/super/order" "github.com/brimdata/super/pkg/field" "github.com/segmentio/ksuid" @@ -31,21 +30,23 @@ type ( Meta string `json:"meta"` } Pool struct { - Kind string `json:"kind"` - Name string `json:"name"` - ID ksuid.KSUID `json:"id"` - Inferred bool `json:"inferred"` + Kind string `json:"kind"` + Name string `json:"name"` + ID ksuid.KSUID `json:"id"` } Path struct { - Kind string `json:"kind"` - URI string `json:"uri"` - Inferred bool `json:"inferred"` + Kind string `json:"kind"` + URI string `json:"uri"` + } + Null struct { + Kind string `json:"kind"` } ) func (*LakeMeta) Source() {} func (*Pool) Source() {} func (*Path) Source() {} +func (*Null) Source() {} type Channel struct { Name string `json:"name"` @@ -53,26 +54,22 @@ type Channel struct { Sort order.SortKeys `json:"sort"` } -func Analyze(ctx context.Context, query string, src *data.Source, head *lakeparse.Commitish) (*Info, error) { +func Analyze(ctx context.Context, query string, src *data.Source) (*Info, error) { ast, err := parser.ParseQuery(query) if err != nil { return nil, err } - entry, err := semantic.Analyze(ctx, ast, src, head) + entry, err := semantic.Analyze(ctx, ast, src, false) if err != nil { return nil, err } - return AnalyzeDAG(ctx, entry, src, head) + return AnalyzeDAG(ctx, entry, src) } -func AnalyzeDAG(ctx context.Context, entry dag.Seq, src *data.Source, head *lakeparse.Commitish) (*Info, error) { - srcInferred := !semantic.HasSource(entry) - if err := semantic.AddDefaultSource(ctx, &entry, src, head); err != nil { - return nil, err - } +func AnalyzeDAG(ctx context.Context, entry dag.Seq, src *data.Source) (*Info, error) { var err error var info Info - if info.Sources, err = describeSources(ctx, src.Lake(), entry[0], srcInferred); err != nil { + if info.Sources, err = describeSources(ctx, src.Lake(), entry[0]); err != nil { return nil, err } sortKeys, err := optimizer.New(ctx, src).SortKeys(entry) @@ -102,14 +99,14 @@ func AnalyzeDAG(ctx context.Context, entry dag.Seq, src *data.Source, head *lake return &info, nil } -func describeSources(ctx context.Context, lk *lake.Root, o dag.Op, inferred bool) ([]Source, error) { +func describeSources(ctx context.Context, lk *lake.Root, o dag.Op) ([]Source, error) { switch o := o.(type) { case *dag.Scope: - return describeSources(ctx, lk, o.Body[0], inferred) + return describeSources(ctx, lk, o.Body[0]) case *dag.Fork: var s []Source for _, p := range o.Paths { - out, err := describeSources(ctx, lk, p[0], inferred) + out, err := describeSources(ctx, lk, p[0]) if err != nil { return nil, err } @@ -117,19 +114,21 @@ func describeSources(ctx context.Context, lk *lake.Root, o dag.Op, inferred bool } return s, nil case *dag.DefaultScan: - return []Source{&Path{Kind: "Path", URI: "stdio://stdin", Inferred: inferred}}, nil + return []Source{&Path{Kind: "Path", URI: "stdio://stdin"}}, nil + case *dag.NullScan: + return []Source{&Null{Kind: "Null"}}, nil case *dag.FileScan: - return []Source{&Path{Kind: "Path", URI: o.Path, Inferred: inferred}}, nil + return []Source{&Path{Kind: "Path", URI: o.Path}}, nil case *dag.HTTPScan: - return []Source{&Path{Kind: "Path", URI: o.URL, Inferred: inferred}}, nil + return []Source{&Path{Kind: "Path", URI: o.URL}}, nil case *dag.PoolScan: - return sourceOfPool(ctx, lk, o.ID, inferred) + return sourceOfPool(ctx, lk, o.ID) case *dag.Lister: - return sourceOfPool(ctx, lk, o.Pool, inferred) + return sourceOfPool(ctx, lk, o.Pool) case *dag.SeqScan: - return sourceOfPool(ctx, lk, o.Pool, inferred) + return sourceOfPool(ctx, lk, o.Pool) case *dag.CommitMetaScan: - return sourceOfPool(ctx, lk, o.Pool, inferred) + return sourceOfPool(ctx, lk, o.Pool) case *dag.LakeMetaScan: return []Source{&LakeMeta{Kind: "LakeMeta", Meta: o.Meta}}, nil default: @@ -137,16 +136,15 @@ func describeSources(ctx context.Context, lk *lake.Root, o dag.Op, inferred bool } } -func sourceOfPool(ctx context.Context, lk *lake.Root, id ksuid.KSUID, inferred bool) ([]Source, error) { +func sourceOfPool(ctx context.Context, lk *lake.Root, id ksuid.KSUID) ([]Source, error) { p, err := lk.OpenPool(ctx, id) if err != nil { return nil, err } return []Source{&Pool{ - Kind: "Pool", - ID: id, - Name: p.Name, - Inferred: inferred, + Kind: "Pool", + ID: id, + Name: p.Name, }}, nil } diff --git a/compiler/file.go b/compiler/file.go index 3c80c08bab..8182e7ddc6 100644 --- a/compiler/file.go +++ b/compiler/file.go @@ -23,7 +23,7 @@ func NewFileSystemCompiler(engine storage.Engine) runtime.Compiler { } func (f *fsCompiler) NewQuery(rctx *runtime.Context, ast *parser.AST, readers []zio.Reader) (runtime.Query, error) { - job, err := NewJob(rctx, ast, f.src, nil) + job, err := NewJob(rctx, ast, f.src, len(readers) > 0) if err != nil { return nil, err } @@ -45,7 +45,7 @@ func (f *fsCompiler) NewQuery(rctx *runtime.Context, ast *parser.AST, readers [] return optimizeAndBuild(job, readers) } -func (*fsCompiler) NewLakeQuery(_ *runtime.Context, ast *parser.AST, parallelism int, head *lakeparse.Commitish) (runtime.Query, error) { +func (*fsCompiler) NewLakeQuery(_ *runtime.Context, ast *parser.AST, parallelism int) (runtime.Query, error) { panic("NewLakeQuery called on compiler.fsCompiler") } diff --git a/compiler/job.go b/compiler/job.go index 15b74f291b..3e7b06c41f 100644 --- a/compiler/job.go +++ b/compiler/job.go @@ -2,6 +2,7 @@ package compiler import ( "errors" + "fmt" "github.com/brimdata/super/compiler/dag" "github.com/brimdata/super/compiler/data" @@ -28,7 +29,7 @@ type Job struct { entry dag.Seq } -func NewJob(rctx *runtime.Context, ast *parser.AST, src *data.Source, head *lakeparse.Commitish) (*Job, error) { +func NewJob(rctx *runtime.Context, ast *parser.AST, src *data.Source, extInput bool) (*Job, error) { // An AST always begins with a Sequential op with at least one // operator. If the first op is a From or a Parallel whose branches // are Sequentials with a leading From, then we presume there is @@ -39,10 +40,7 @@ func NewJob(rctx *runtime.Context, ast *parser.AST, src *data.Source, head *lake // caller via runtime.readers. In most cases, the AST is left // with an ast.From at the entry point, and hence a dag.From for the // DAG's entry point. - if len(ast.Parsed()) == 0 { - return nil, errors.New("internal error: AST seq cannot be empty") - } - entry, err := semantic.AnalyzeAddSource(rctx.Context, ast, src, head) + entry, err := semantic.Analyze(rctx.Context, ast, src, extInput) if err != nil { return nil, err } @@ -131,10 +129,14 @@ func VectorCompile(rctx *runtime.Context, query string, object *vcache.Object) ( return nil, err } src := &data.Source{} - entry, err := semantic.Analyze(rctx.Context, ast, src, nil) + entry, err := semantic.Analyze(rctx.Context, ast, src, true) if err != nil { return nil, err } + if _, ok := entry[0].(*dag.DefaultScan); !ok { + panic("DAG assumptions violated") + } + entry = entry[1:] puller := vam.NewVectorProjection(rctx.Zctx, object, nil) //XXX project all builder := kernel.NewBuilder(rctx, src) outputs, err := builder.BuildWithPuller(entry, puller) @@ -153,7 +155,7 @@ func VectorFilterCompile(rctx *runtime.Context, query string, src *data.Source, // Eventually the semantic analyzer + kernel will resolve the pool but // for now just do this manually. if !src.IsLake() { - return nil, errors.New("non-lake searches not supported") + return nil, errors.New("non-lake vectorized search not supported") } poolID, err := src.PoolID(rctx.Context, head.Pool) if err != nil { @@ -163,18 +165,23 @@ func VectorFilterCompile(rctx *runtime.Context, query string, src *data.Source, if err != nil { return nil, err } - ast, err := parser.ParseQuery(query) + spec, err := head.FromSpec("") + if err != nil { + return nil, err + } + ast, err := parser.ParseQuery(fmt.Sprintf("%s | %s", spec, query)) if err != nil { return nil, err } - entry, err := semantic.Analyze(rctx.Context, ast, src, nil) + entry, err := semantic.Analyze(rctx.Context, ast, src, false) if err != nil { return nil, err } - if len(entry) != 2 { + // from -> filter -> output + if len(entry) != 3 { return nil, errors.New("filter query must have a single op") } - f, ok := entry[0].(*dag.Filter) + f, ok := entry[1].(*dag.Filter) if !ok { return nil, errors.New("filter query must be a single filter op") } diff --git a/compiler/kernel/op.go b/compiler/kernel/op.go index 876556d838..f39793445e 100644 --- a/compiler/kernel/op.go +++ b/compiler/kernel/op.go @@ -287,6 +287,9 @@ func (b *Builder) compileLeaf(o dag.Op, parent zbuf.Puller) (zbuf.Puller, error) scanners = append(scanners, scanner) } return zbuf.MultiScanner(scanners...), nil + case *dag.NullScan: + //XXX we need something that implements the done protocol and restarst + return zbuf.NewPuller(zbuf.NewArray([]super.Value{super.Null})), nil case *dag.Lister: if parent != nil { return nil, errors.New("internal error: data source cannot have a parent operator") @@ -744,7 +747,7 @@ func isEntry(seq dag.Seq) bool { return false } switch op := seq[0].(type) { - case *dag.Lister, *dag.DefaultScan, *dag.FileScan, *dag.HTTPScan, *dag.PoolScan, *dag.LakeMetaScan, *dag.PoolMetaScan, *dag.CommitMetaScan: + case *dag.Lister, *dag.DefaultScan, *dag.FileScan, *dag.HTTPScan, *dag.PoolScan, *dag.LakeMetaScan, *dag.PoolMetaScan, *dag.CommitMetaScan, *dag.NullScan: return true case *dag.Scope: return isEntry(op.Body) diff --git a/compiler/lake.go b/compiler/lake.go index 1287f88e0c..3f4694e8e3 100644 --- a/compiler/lake.go +++ b/compiler/lake.go @@ -31,8 +31,8 @@ func NewLakeCompiler(r *lake.Root) runtime.Compiler { return &lakeCompiler{src: data.NewSource(storage.NewRemoteEngine(), r)} } -func (l *lakeCompiler) NewLakeQuery(rctx *runtime.Context, ast *parser.AST, parallelism int, head *lakeparse.Commitish) (runtime.Query, error) { - job, err := NewJob(rctx, ast, l.src, head) +func (l *lakeCompiler) NewLakeQuery(rctx *runtime.Context, ast *parser.AST, parallelism int) (runtime.Query, error) { + job, err := NewJob(rctx, ast, l.src, false) if err != nil { return nil, err } @@ -57,7 +57,7 @@ func (l *lakeCompiler) NewLakeQuery(rctx *runtime.Context, ast *parser.AST, para } func (l *lakeCompiler) NewLakeDeleteQuery(rctx *runtime.Context, ast *parser.AST, head *lakeparse.Commitish) (runtime.DeleteQuery, error) { - job, err := newDeleteJob(rctx, ast, l.src, head) + job, err := newDeleteJob(rctx, ast, head.Pool, head.Branch, l.src) if err != nil { return nil, err } @@ -76,15 +76,15 @@ func (InvalidDeleteWhereQuery) Error() string { return "invalid delete where query: must be a single filter operation" } -func newDeleteJob(rctx *runtime.Context, parsed *parser.AST, src *data.Source, head *lakeparse.Commitish) (*Job, error) { - if err := parsed.ConvertToDeleteWhere(); err != nil { +func newDeleteJob(rctx *runtime.Context, parsed *parser.AST, pool, branch string, src *data.Source) (*Job, error) { + if err := parsed.ConvertToDeleteWhere(pool, branch); err != nil { return nil, err } seq := parsed.Parsed() if len(seq) != 2 { return nil, &InvalidDeleteWhereQuery{} } - entry, err := semantic.Analyze(rctx.Context, parsed, src, head) + entry, err := semantic.Analyze(rctx.Context, parsed, src, false) if err != nil { return nil, err } diff --git a/compiler/parser/api.go b/compiler/parser/api.go index 72c1dcf0e1..5a47cd2ccc 100644 --- a/compiler/parser/api.go +++ b/compiler/parser/api.go @@ -24,11 +24,15 @@ func (a *AST) Files() *srcfiles.List { return a.files } -func (a *AST) ConvertToDeleteWhere() error { +func (a *AST) ConvertToDeleteWhere(pool, branch string) error { if len(a.seq) == 0 { return errors.New("internal error: AST seq cannot be empty") } - a.seq.Prepend(&ast.Delete{Kind: "Delete"}) + a.seq.Prepend(&ast.Delete{ + Kind: "Delete", + Pool: pool, + Branch: branch, + }) return nil } @@ -39,6 +43,9 @@ func ParseQuery(query string, filenames ...string) (*AST, error) { if err != nil { return nil, err } + if files.Text == "" { + return &AST{files: files}, nil + } p, err := Parse("", []byte(files.Text), Recover(false)) if err != nil { if err := convertParseErrs(err, files); err != nil { diff --git a/compiler/reader.go b/compiler/reader.go index c601bc6240..a308ae8529 100644 --- a/compiler/reader.go +++ b/compiler/reader.go @@ -21,7 +21,7 @@ func (i *anyCompiler) NewQuery(rctx *runtime.Context, ast *parser.AST, readers [ if len(readers) != 1 { return nil, fmt.Errorf("NewQuery: Zed program expected %d readers", len(readers)) } - job, err := NewJob(rctx, ast, data.NewSource(nil, nil), nil) + job, err := NewJob(rctx, ast, data.NewSource(nil, nil), true) if err != nil { return nil, err } @@ -30,7 +30,7 @@ func (i *anyCompiler) NewQuery(rctx *runtime.Context, ast *parser.AST, readers [ // XXX currently used only by group-by test, need to deprecate func CompileWithSortKey(rctx *runtime.Context, ast *parser.AST, r zio.Reader, sortKey order.SortKey) (*exec.Query, error) { - job, err := NewJob(rctx, ast, data.NewSource(nil, nil), nil) + job, err := NewJob(rctx, ast, data.NewSource(nil, nil), true) if err != nil { return nil, err } @@ -42,7 +42,7 @@ func CompileWithSortKey(rctx *runtime.Context, ast *parser.AST, r zio.Reader, so return optimizeAndBuild(job, []zio.Reader{r}) } -func (*anyCompiler) NewLakeQuery(rctx *runtime.Context, ast *parser.AST, parallelism int, head *lakeparse.Commitish) (runtime.Query, error) { +func (*anyCompiler) NewLakeQuery(rctx *runtime.Context, ast *parser.AST, parallelism int) (runtime.Query, error) { panic("NewLakeQuery called on compiler.anyCompiler") } diff --git a/compiler/semantic/analyzer.go b/compiler/semantic/analyzer.go index 9a2752b239..7726822ce0 100644 --- a/compiler/semantic/analyzer.go +++ b/compiler/semantic/analyzer.go @@ -10,42 +10,36 @@ import ( "github.com/brimdata/super/compiler/data" "github.com/brimdata/super/compiler/parser" "github.com/brimdata/super/compiler/srcfiles" - "github.com/brimdata/super/lakeparse" ) // Analyze performs a semantic analysis of the AST, translating it from AST // to DAG form, resolving syntax ambiguities, and performing constant propagation. // After semantic analysis, the DAG is ready for either optimization or compilation. -func Analyze(ctx context.Context, ast *parser.AST, source *data.Source, head *lakeparse.Commitish) (dag.Seq, error) { +func Analyze(ctx context.Context, ast *parser.AST, source *data.Source, extInput bool) (dag.Seq, error) { files := ast.Files() - a := newAnalyzer(ctx, files, source, head) - s := a.semSeq(ast.Parsed()) - s = a.checkOutputs(true, s) - return s, files.Error() -} - -// AnalyzeAddSource is the same as Analyze but it adds a default source if the -// DAG does not have one. -func AnalyzeAddSource(ctx context.Context, ast *parser.AST, source *data.Source, head *lakeparse.Commitish) (dag.Seq, error) { - files := ast.Files() - a := newAnalyzer(ctx, files, source, head) - s := a.semSeq(ast.Parsed()) - s = a.checkOutputs(true, s) - if err := files.Error(); err != nil { - return nil, err - } - if !HasSource(s) { - if err := AddDefaultSource(ctx, &s, source, head); err != nil { - return nil, err + a := newAnalyzer(ctx, files, source) + seq := a.semSeq(ast.Parsed()) + if !HasSource(seq) { + if a.source.IsLake() { + if len(seq) == 0 { + return nil, errors.New("query text is missing") + } + seq.Prepend(&dag.NullScan{Kind: "NullScan"}) + } else if extInput { + seq.Prepend(&dag.DefaultScan{Kind: "DefaultScan"}) + } else { + // This is a local query and there's no external input + // (i.e., no command-line file args) + seq.Prepend(&dag.NullScan{Kind: "NullScan"}) } } - return s, nil + seq = a.checkOutputs(true, seq) + return seq, files.Error() } type analyzer struct { ctx context.Context files *srcfiles.List - head *lakeparse.Commitish opStack []*ast.OpDecl outputs map[*dag.Output]ast.Node source *data.Source @@ -53,11 +47,10 @@ type analyzer struct { zctx *super.Context } -func newAnalyzer(ctx context.Context, files *srcfiles.List, source *data.Source, head *lakeparse.Commitish) *analyzer { +func newAnalyzer(ctx context.Context, files *srcfiles.List, source *data.Source) *analyzer { return &analyzer{ ctx: ctx, files: files, - head: head, outputs: make(map[*dag.Output]ast.Node), source: source, scope: NewScope(nil), @@ -66,8 +59,11 @@ func newAnalyzer(ctx context.Context, files *srcfiles.List, source *data.Source, } func HasSource(seq dag.Seq) bool { + if len(seq) == 0 { + return false + } switch op := seq[0].(type) { - case *dag.FileScan, *dag.HTTPScan, *dag.PoolScan, *dag.LakeMetaScan, *dag.PoolMetaScan, *dag.CommitMetaScan, *dag.DeleteScan: + case *dag.FileScan, *dag.HTTPScan, *dag.PoolScan, *dag.LakeMetaScan, *dag.PoolMetaScan, *dag.CommitMetaScan, *dag.DeleteScan, *dag.NullScan: return true case *dag.Fork: return HasSource(op.Paths[0]) @@ -77,34 +73,6 @@ func HasSource(seq dag.Seq) bool { return false } -func AddDefaultSource(ctx context.Context, seq *dag.Seq, source *data.Source, head *lakeparse.Commitish) error { - if HasSource(*seq) { - return nil - } - // No from so add a source. - if head == nil { - seq.Prepend(&dag.DefaultScan{Kind: "DefaultScan"}) - return nil - } - // Verify pool exists for HEAD - if _, err := source.PoolID(ctx, head.Pool); err != nil { - return err - } - fromHead := &ast.From{ - Kind: "From", - Elems: []*ast.FromElem{{ - Kind: "FromElem", - Entity: &ast.Name{ - Kind: "Name", - Text: "HEAD", - }, - }}, - } - headSeq := newAnalyzer(ctx, &srcfiles.List{}, source, head).semFrom(fromHead, nil) - seq.Prepend(headSeq[0]) - return nil -} - func (a *analyzer) enterScope() { a.scope = NewScope(a.scope) } diff --git a/compiler/semantic/op.go b/compiler/semantic/op.go index 9bdd53f44a..77d8a87ce8 100644 --- a/compiler/semantic/op.go +++ b/compiler/semantic/op.go @@ -339,14 +339,6 @@ func (a *analyzer) semPool(nameLoc ast.Node, poolName string, args *ast.PoolArgs meta = nullableName(args.Meta) tap = args.Tap } - if poolName == "HEAD" { - if a.head == nil || a.head.Pool == "" { - a.error(nameLoc, errors.New("cannot resolve unknown HEAD")) - return badOp() - } - poolName = a.head.Pool - commit = a.head.Branch - } poolID, err := a.source.PoolID(a.ctx, poolName) if err != nil { a.error(nameLoc, err) @@ -422,20 +414,16 @@ func (a *analyzer) semDelete(op *ast.Delete) dag.Op { a.error(op, errors.New("deletion requires data lake")) return badOp() } - // delete-where only supports deleting at head - pool := a.head.Pool - commit := a.head.Branch - - poolID, err := a.source.PoolID(a.ctx, pool) + poolID, err := a.source.PoolID(a.ctx, op.Pool) if err != nil { a.error(op, err) return badOp() } var commitID ksuid.KSUID - if commit != "" { + if op.Branch != "" { var err error - if commitID, err = lakeparse.ParseID(commit); err != nil { - commitID, err = a.source.CommitObject(a.ctx, poolID, commit) + if commitID, err = lakeparse.ParseID(op.Branch); err != nil { + commitID, err = a.source.CommitObject(a.ctx, poolID, op.Branch) if err != nil { a.error(op, err) return badOp() diff --git a/compiler/srcfiles/list.go b/compiler/srcfiles/list.go index 069ffb4bf6..0f95f5e09e 100644 --- a/compiler/srcfiles/list.go +++ b/compiler/srcfiles/list.go @@ -42,9 +42,6 @@ func Concat(filenames []string, query string) (*List, error) { b.Write(bb) b.WriteByte('\n') } - if b.Len() == 0 && query == "" { - query = "*" - } // Empty string is the unnamed query text while the included files all // have names. files = append(files, newFile("", b.Len(), []byte(query))) diff --git a/compiler/ztests/head.yaml b/compiler/ztests/head.yaml index 4bb728d6cd..38f8f1d536 100644 --- a/compiler/ztests/head.yaml +++ b/compiler/ztests/head.yaml @@ -9,11 +9,11 @@ script: | outputs: - name: stdout data: | - reader + null | head 1 | output main === - reader + null | ( const x = 1 diff --git a/compiler/ztests/merge-filters.yaml b/compiler/ztests/merge-filters.yaml index c9a62b4f49..609bb3c533 100644 --- a/compiler/ztests/merge-filters.yaml +++ b/compiler/ztests/merge-filters.yaml @@ -1,5 +1,5 @@ script: | - super compile -C -O 'where a | where b' + super compile -C -O 'from foo | where a | where b' echo === super compile -C -O 'from ( file a => where b | where c file d => where e | where f ) | where g' echo === @@ -10,7 +10,7 @@ script: | outputs: - name: stdout data: | - reader filter (a and b) + file foo filter (a and b) | output main === fork ( @@ -21,13 +21,13 @@ outputs: ) | output main === - reader + null | over a => ( where b and c ) | output main === - reader + null | fork ( => where a and b diff --git a/compiler/ztests/remove-passops.yaml b/compiler/ztests/remove-passops.yaml index 2dc99db9bf..a1ffab814e 100644 --- a/compiler/ztests/remove-passops.yaml +++ b/compiler/ztests/remove-passops.yaml @@ -1,7 +1,7 @@ -script: super compile -O -C 'x>1 | pass | pass | x>2 | pass' +script: super compile -O -C 'from foo | x>1 | pass | pass | x>2 | pass' outputs: - name: stdout data: | - reader filter (x>1 and x>2) + file foo filter (x>1 and x>2) | output main diff --git a/compiler/ztests/tail.yaml b/compiler/ztests/tail.yaml index aefff3b78a..2d129e46ed 100644 --- a/compiler/ztests/tail.yaml +++ b/compiler/ztests/tail.yaml @@ -9,11 +9,11 @@ script: | outputs: - name: stdout data: | - reader + null | tail 1 | output main === - reader + null | ( const x = 1 diff --git a/compiler/ztests/udf-implied-where.yaml b/compiler/ztests/udf-implied-where.yaml index 6febdd3383..079ef66ff4 100644 --- a/compiler/ztests/udf-implied-where.yaml +++ b/compiler/ztests/udf-implied-where.yaml @@ -4,7 +4,7 @@ script: | outputs: - name: stdout data: | - reader + null | ( func h(e): ( has(e) diff --git a/docs/libraries/go.md b/docs/libraries/go.md index 712d45f723..fed92aeb93 100644 --- a/docs/libraries/go.md +++ b/docs/libraries/go.md @@ -134,7 +134,7 @@ func main() { if err != nil { log.Fatalln(err) } - q, err := lake.Query(ctx, nil, "from Demo") + q, err := lake.Query(ctx, "from Demo") if err != nil { log.Fatalln(err) } diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go index 383cca5e11..7e29d90cd8 100644 --- a/fuzz/fuzz.go +++ b/fuzz/fuzz.go @@ -109,7 +109,7 @@ func RunQuery(t testing.TB, zctx *super.Context, readers []zio.Reader, querySour // Infer demand // TODO This is a hack and should be replaced by a cleaner interface in CompileQuery. source := data.NewSource(engine, nil) - dag, err := semantic.AnalyzeAddSource(ctx, ast, source, nil) + dag, err := semantic.Analyze(ctx, ast, source, true) if err != nil { t.Skipf("%v", err) } diff --git a/lake/api/api.go b/lake/api/api.go index 8b1e260b3e..4c8ed8509b 100644 --- a/lake/api/api.go +++ b/lake/api/api.go @@ -11,7 +11,6 @@ import ( "github.com/brimdata/super/api/client" "github.com/brimdata/super/lake" "github.com/brimdata/super/lake/pools" - "github.com/brimdata/super/lakeparse" "github.com/brimdata/super/order" "github.com/brimdata/super/zbuf" "github.com/brimdata/super/zio" @@ -22,7 +21,7 @@ import ( type Interface interface { Root() *lake.Root - Query(ctx context.Context, head *lakeparse.Commitish, src string, srcfiles ...string) (zbuf.Scanner, error) + Query(ctx context.Context, src string, srcfiles ...string) (zbuf.Scanner, error) PoolID(ctx context.Context, poolName string) (ksuid.KSUID, error) CommitObject(ctx context.Context, poolID ksuid.KSUID, branchName string) (ksuid.KSUID, error) CreatePool(context.Context, string, order.SortKeys, int, int64) (ksuid.KSUID, error) @@ -55,7 +54,7 @@ func IsLakeService(u string) bool { func LookupPoolByName(ctx context.Context, api Interface, name string) (*pools.Config, error) { b := newBuffer(pools.Config{}) zed := fmt.Sprintf("from :pools | name == '%s'", name) - q, err := api.Query(ctx, nil, zed) + q, err := api.Query(ctx, zed) if err != nil { return nil, err } @@ -79,7 +78,7 @@ func LookupPoolByName(ctx context.Context, api Interface, name string) (*pools.C func GetPools(ctx context.Context, api Interface) ([]*pools.Config, error) { b := newBuffer(pools.Config{}) - q, err := api.Query(ctx, nil, "from :pools") + q, err := api.Query(ctx, "from :pools") if err != nil { return nil, err } @@ -97,7 +96,7 @@ func GetPools(ctx context.Context, api Interface) ([]*pools.Config, error) { func LookupPoolByID(ctx context.Context, api Interface, id ksuid.KSUID) (*pools.Config, error) { b := newBuffer(pools.Config{}) zed := fmt.Sprintf("from :pools | id == hex('%s')", idToHex(id)) - q, err := api.Query(ctx, nil, zed) + q, err := api.Query(ctx, zed) if err != nil { return nil, err } @@ -122,7 +121,7 @@ func LookupPoolByID(ctx context.Context, api Interface, id ksuid.KSUID) (*pools. func LookupBranchByName(ctx context.Context, api Interface, poolName, branchName string) (*lake.BranchMeta, error) { b := newBuffer(lake.BranchMeta{}) zed := fmt.Sprintf("from :branches | pool.name == '%s' branch.name == '%s'", poolName, branchName) - q, err := api.Query(ctx, nil, zed) + q, err := api.Query(ctx, zed) if err != nil { return nil, err } @@ -147,7 +146,7 @@ func LookupBranchByName(ctx context.Context, api Interface, poolName, branchName func LookupBranchByID(ctx context.Context, api Interface, id ksuid.KSUID) (*lake.BranchMeta, error) { b := newBuffer(lake.BranchMeta{}) zed := fmt.Sprintf("from :branches | branch.id == 'hex(%s)'", idToHex(id)) - q, err := api.Query(ctx, nil, zed) + q, err := api.Query(ctx, zed) if err != nil { return nil, err } diff --git a/lake/api/local.go b/lake/api/local.go index 7cb0091a13..0432c16882 100644 --- a/lake/api/local.go +++ b/lake/api/local.go @@ -105,12 +105,12 @@ func (l *local) Compact(ctx context.Context, poolID ksuid.KSUID, branchName stri return exec.Compact(ctx, l.root, pool, branchName, objects, writeVectors, commit.Author, commit.Body, commit.Meta) } -func (l *local) Query(ctx context.Context, head *lakeparse.Commitish, src string, srcfiles ...string) (zbuf.Scanner, error) { +func (l *local) Query(ctx context.Context, src string, srcfiles ...string) (zbuf.Scanner, error) { ast, err := parser.ParseQuery(src, srcfiles...) if err != nil { return nil, err } - q, err := runtime.CompileLakeQuery(ctx, super.NewContext(), l.compiler, ast, head) + q, err := runtime.CompileLakeQuery(ctx, super.NewContext(), l.compiler, ast) if err != nil { return nil, err } diff --git a/lake/api/remote.go b/lake/api/remote.go index d2e5a61856..9bca3a1895 100644 --- a/lake/api/remote.go +++ b/lake/api/remote.go @@ -119,8 +119,8 @@ func (r *remote) Revert(ctx context.Context, poolID ksuid.KSUID, branchName stri return res.Commit, err } -func (r *remote) Query(ctx context.Context, head *lakeparse.Commitish, src string, srcfiles ...string) (zbuf.Scanner, error) { - res, err := r.conn.Query(ctx, head, src, srcfiles...) +func (r *remote) Query(ctx context.Context, src string, srcfiles ...string) (zbuf.Scanner, error) { + res, err := r.conn.Query(ctx, src, srcfiles...) if err != nil { return nil, err } diff --git a/lake/ztests/checkout-dash-p.yaml b/lake/ztests/checkout-dash-p.yaml index 4acddd4844..013f4b9fc3 100644 --- a/lake/ztests/checkout-dash-p.yaml +++ b/lake/ztests/checkout-dash-p.yaml @@ -7,8 +7,7 @@ script: | super db branch -q child super db use -q @child super db load -q b.jsup - super db use -q POOL - super db query -z "*" + super db query -z "from POOL" inputs: - name: a.jsup diff --git a/lake/ztests/debug.yaml b/lake/ztests/debug.yaml index 628d5c5594..cc0463e40f 100644 --- a/lake/ztests/debug.yaml +++ b/lake/ztests/debug.yaml @@ -3,7 +3,7 @@ script: | super db init -q super db create -use -q test echo '{x: "foo"}' | super db load -q - - super db query -z 'debug f"debug: {x}"' + super db query -z 'from test | debug f"debug: {x}"' outputs: - name: stdout diff --git a/lake/ztests/delete-where-missing.yaml b/lake/ztests/delete-where-missing.yaml index b0d2802af3..d4b89e3d1b 100644 --- a/lake/ztests/delete-where-missing.yaml +++ b/lake/ztests/delete-where-missing.yaml @@ -4,9 +4,9 @@ script: | super db create -use -q test super db load -q in.jsup ! super db delete -q -where 'uid==C3UeSqaSOFRReHD68' - super db query -z 'count:=count()' + super db query -z 'from test | count:=count()' super db delete -q -where 'uid=="C3UeSqaSOFRReHD68"' - super db query -z 'count:=count()' + super db query -z 'from test | count:=count()' inputs: - name: in.jsup diff --git a/lake/ztests/delete-where.yaml b/lake/ztests/delete-where.yaml index b7ad6fc86a..4cb29157d1 100644 --- a/lake/ztests/delete-where.yaml +++ b/lake/ztests/delete-where.yaml @@ -6,9 +6,9 @@ script: | super db create -use -q -S 1KB -orderby ts:$order test seq 1000 | super -c '{ts:this-1,s:f"val{this-1}"}' - | super db load -q - super db delete -q -where 'ts > 400 and ts <= 500' - super db query -z 'count:=count()' + super db query -z 'from test | count:=count()' super db delete -q -where 's == "val1" or s == "val999"' - super db query -z 'count:=count()' + super db query -z 'from test | count:=count()' super db drop -f -q test done diff --git a/lake/ztests/delete.yaml b/lake/ztests/delete.yaml index 722ea78a33..f0161fe04d 100644 --- a/lake/ztests/delete.yaml +++ b/lake/ztests/delete.yaml @@ -5,10 +5,10 @@ script: | super db load -q 1.jsup id=$(super db query -f text "from test@main:objects | cut id:=ksuid(id) | tail 1") super db load -q 2.jsup - super db query -z "*" + super db query -z "from test" echo === | tee /dev/stderr super db delete -q $id - super db query -z "*" + super db query -z "from test" echo === | tee /dev/stderr ! super db delete -q 27aaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/lake/ztests/query-error.yaml b/lake/ztests/query-error.yaml index 9c7e14ed3d..ba6856b4c7 100644 --- a/lake/ztests/query-error.yaml +++ b/lake/ztests/query-error.yaml @@ -9,8 +9,8 @@ script: | outputs: - name: stderr data: | - query must include a 'from' operator - cannot resolve unknown HEAD at line 1, column 6: + query text is missing + HEAD: pool not found at line 1, column 6: from HEAD ~~~~ unknown lake metadata type "unknownmeta" in from operator at line 1, column 6: diff --git a/lake/ztests/revert-revert.yaml b/lake/ztests/revert-revert.yaml index c7f3f865ee..4a4663e50f 100644 --- a/lake/ztests/revert-revert.yaml +++ b/lake/ztests/revert-revert.yaml @@ -7,10 +7,10 @@ script: | super db query -z "from test | sort this" r=$(super db revert $a | awk '{print $5}') echo === - super db query -z "sort this" + super db query -z "from test | sort this" super db revert -q $r echo === - super db query -z "sort this" + super db query -z "from test | sort this" inputs: - name: a.jsup diff --git a/lake/ztests/revert.yaml b/lake/ztests/revert.yaml index bdf20e7f4a..0eda7088ae 100644 --- a/lake/ztests/revert.yaml +++ b/lake/ztests/revert.yaml @@ -4,13 +4,13 @@ script: | super db create -use -q test a=$(super db load a.jsup | head -1 | awk '{print $1}') b=$(super db load b.jsup | head -1 | awk '{print $1}') - super db query -z "sort this" + super db query -z "from test | sort this" super db revert -q $a echo === - super db query -z "sort this" + super db query -z "from test | sort this" super db revert -q $b echo === - super db query -z "sort this" + super db query -z "from test | sort this" inputs: - name: a.jsup diff --git a/lake/ztests/seek-index-boundaries.yaml b/lake/ztests/seek-index-boundaries.yaml index 92ba896fb5..ac21413213 100644 --- a/lake/ztests/seek-index-boundaries.yaml +++ b/lake/ztests/seek-index-boundaries.yaml @@ -2,14 +2,13 @@ script: | export SUPER_DB_LAKE=test super db init -q for o in asc desc; do - echo // $o | tee /dev/stderr super db create -use -q -seekstride 1B -orderby ts:$o $o seq 20 | super -c 'yield {ts:this}' - | super db load -q - - source query.sh 'ts == 5' - source query.sh 'ts < 2' - source query.sh 'ts <= 2' - source query.sh 'ts > 19' - source query.sh 'ts >= 19' + source query.sh "from $o | ts == 5" + source query.sh "from $o | ts < 2" + source query.sh "from $o | ts <= 2" + source query.sh "from $o | ts > 19" + source query.sh "from $o | ts >= 19" done inputs: @@ -20,53 +19,49 @@ inputs: outputs: - name: stdout data: | - // asc - // ts == 5 + // from asc | ts == 5 {ts:5} - // ts < 2 + // from asc | ts < 2 {ts:1} - // ts <= 2 + // from asc | ts <= 2 {ts:1} {ts:2} - // ts > 19 + // from asc | ts > 19 {ts:20} - // ts >= 19 + // from asc | ts >= 19 {ts:19} {ts:20} - // desc - // ts == 5 + // from desc | ts == 5 {ts:5} - // ts < 2 + // from desc | ts < 2 {ts:1} - // ts <= 2 + // from desc | ts <= 2 {ts:2} {ts:1} - // ts > 19 + // from desc | ts > 19 {ts:20} - // ts >= 19 + // from desc | ts >= 19 {ts:20} {ts:19} - name: stderr data: | - // asc - // ts == 5 + // from asc | ts == 5 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts < 2 + // from asc | ts < 2 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts <= 2 + // from asc | ts <= 2 {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} - // ts > 19 + // from asc | ts > 19 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts >= 19 + // from asc | ts >= 19 {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} - // desc - // ts == 5 + // from desc | ts == 5 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts < 2 + // from desc | ts < 2 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts <= 2 + // from desc | ts <= 2 {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} - // ts > 19 + // from desc | ts > 19 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts >= 19 + // from desc | ts >= 19 {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} diff --git a/lake/ztests/seek-index-null.yaml b/lake/ztests/seek-index-null.yaml index 9b8fafaa57..74190af99a 100644 --- a/lake/ztests/seek-index-null.yaml +++ b/lake/ztests/seek-index-null.yaml @@ -5,7 +5,7 @@ script: | echo // $o | tee /dev/stderr super db create -q -seekstride 2KB -orderby ts:$o $o super db load -q -use $o babble.jsup null.jsup - super db query -z -s -use $o "ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" + super db query -z -s "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" done inputs: diff --git a/lake/ztests/seek-index-ts.yaml b/lake/ztests/seek-index-ts.yaml index 4ed5706bf9..f4b884de33 100644 --- a/lake/ztests/seek-index-ts.yaml +++ b/lake/ztests/seek-index-ts.yaml @@ -2,12 +2,11 @@ script: | export SUPER_DB_LAKE=test super db init -q for o in asc desc; do - echo // $o | tee /dev/stderr super db create -use -seekstride 2KB -orderby ts:$o -q $o super db load -q babble.jsup - source query.sh "ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" - source query.sh "ts == 2020-04-21T23:59:26.06326664Z" - source query.sh "ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar'" + source query.sh "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" + source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z" + source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar'" done @@ -22,37 +21,33 @@ inputs: outputs: - name: stdout data: | - // asc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - // ts == 2020-04-21T23:59:26.06326664Z + // from asc | ts == 2020-04-21T23:59:26.06326664Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // desc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z + // from desc | ts == 2020-04-21T23:59:26.06326664Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - name: stderr data: | - // asc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // ts == 2020-04-21T23:59:26.06326664Z + // from asc | ts == 2020-04-21T23:59:26.06326664Z {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} - // desc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // ts == 2020-04-21T23:59:26.06326664Z + // from desc | ts == 2020-04-21T23:59:26.06326664Z {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} diff --git a/lake/ztests/vacuum.yaml b/lake/ztests/vacuum.yaml index 0a1dfd3fd4..7d49ddff1f 100644 --- a/lake/ztests/vacuum.yaml +++ b/lake/ztests/vacuum.yaml @@ -8,7 +8,7 @@ script: | super db vacuum -dryrun super db vacuum -f super db revert -q $r - ! super db query -z '*' + ! super db query -z 'from test' outputs: - name: stdout diff --git a/runtime/compiler.go b/runtime/compiler.go index 2d58046b01..bb5353663b 100644 --- a/runtime/compiler.go +++ b/runtime/compiler.go @@ -14,7 +14,7 @@ import ( type Compiler interface { NewQuery(*Context, *parser.AST, []zio.Reader) (Query, error) - NewLakeQuery(*Context, *parser.AST, int, *lakeparse.Commitish) (Query, error) + NewLakeQuery(*Context, *parser.AST, int) (Query, error) NewLakeDeleteQuery(*Context, *parser.AST, *lakeparse.Commitish) (DeleteQuery, error) } @@ -44,9 +44,9 @@ func CompileQuery(ctx context.Context, zctx *super.Context, c Compiler, ast *par return q, nil } -func CompileLakeQuery(ctx context.Context, zctx *super.Context, c Compiler, ast *parser.AST, head *lakeparse.Commitish) (Query, error) { +func CompileLakeQuery(ctx context.Context, zctx *super.Context, c Compiler, ast *parser.AST) (Query, error) { rctx := NewContext(ctx, zctx) - q, err := c.NewLakeQuery(rctx, ast, 0, head) + q, err := c.NewLakeQuery(rctx, ast, 0) if err != nil { rctx.Cancel() return nil, err diff --git a/runtime/sam/expr/filter_test.go b/runtime/sam/expr/filter_test.go index 0e597634ca..ccdacc5103 100644 --- a/runtime/sam/expr/filter_test.go +++ b/runtime/sam/expr/filter_test.go @@ -7,6 +7,7 @@ import ( "github.com/brimdata/super" "github.com/brimdata/super/compiler" + "github.com/brimdata/super/compiler/data" "github.com/brimdata/super/compiler/parser" "github.com/brimdata/super/runtime" "github.com/brimdata/super/runtime/sam/expr" @@ -54,7 +55,7 @@ func runCasesHelper(t *testing.T, record string, cases []testcase, expectBufferF t.Helper() ast, err := parser.ParseQuery(c.filter) require.NoError(t, err, "filter: %q", c.filter) - job, err := compiler.NewJob(runtime.DefaultContext(), ast, nil, nil) + job, err := compiler.NewJob(runtime.DefaultContext(), ast, &data.Source{}, true) require.NoError(t, err, "filter: %q", c.filter) err = job.Optimize() require.NoError(t, err, "filter: %q", c.filter) diff --git a/runtime/sam/op/meta/ztests/nulls-max.yaml b/runtime/sam/op/meta/ztests/nulls-max.yaml index 50387e11c8..9a3806c29f 100644 --- a/runtime/sam/op/meta/ztests/nulls-max.yaml +++ b/runtime/sam/op/meta/ztests/nulls-max.yaml @@ -11,7 +11,7 @@ script: | echo '{ts:1}' | super db load -q - super db query -z "from $o:objects | drop id, size" echo "// ===" - super db query -z 'head 1' + super db query -z "from $o | head 1" done outputs: diff --git a/service/auth_test.go b/service/auth_test.go index 0001ced203..55f57a4333 100644 --- a/service/auth_test.go +++ b/service/auth_test.go @@ -37,7 +37,7 @@ func TestAuthIdentity(t *testing.T) { core, conn := newCoreWithConfig(t, service.Config{ Auth: authConfig, }) - _, err := conn.Query(context.Background(), nil, "from [pools]") + _, err := conn.Query(context.Background(), "from [pools]") require.Error(t, err) require.Equal(t, 1.0, promCounterValue(core.Registry(), "request_errors_unauthorized_total")) @@ -59,7 +59,7 @@ func TestAuthIdentity(t *testing.T) { UserID: "test_user_id", }, res) - _, err = conn.Query(context.Background(), nil, "from :pools") + _, err = conn.Query(context.Background(), "from :pools") require.NoError(t, err) } diff --git a/service/client_test.go b/service/client_test.go index 61333ee991..372d4aabb4 100644 --- a/service/client_test.go +++ b/service/client_test.go @@ -48,7 +48,7 @@ func (c *testClient) TestBranchGet(id ksuid.KSUID) (config lake.BranchMeta) { } func (c *testClient) TestPoolList() []pools.Config { - r, err := c.Query(context.Background(), nil, "from :pools") + r, err := c.Query(context.Background(), "from :pools") require.NoError(c, err) defer r.Body.Close() var confs []pools.Config @@ -80,7 +80,7 @@ func (c *testClient) TestBranchPost(poolID ksuid.KSUID, payload api.BranchPostRe } func (c *testClient) TestQuery(query string) string { - r, err := c.Connection.Query(context.Background(), nil, query) + r, err := c.Connection.Query(context.Background(), query) require.NoError(c, err) defer r.Body.Close() zr := zngio.NewReader(super.NewContext(), r.Body) diff --git a/service/handlers.go b/service/handlers.go index 08da636bb6..a27ede26d1 100644 --- a/service/handlers.go +++ b/service/handlers.go @@ -62,7 +62,7 @@ func handleQuery(c *Core, w *ResponseWriter, r *Request) { w.Error(srverr.ErrInvalid(err)) return } - flowgraph, err := runtime.CompileLakeQuery(r.Context(), super.NewContext(), c.compiler, ast, &req.Head) + flowgraph, err := runtime.CompileLakeQuery(r.Context(), super.NewContext(), c.compiler, ast) if err != nil { w.Error(srverr.ErrInvalid(err)) return @@ -182,7 +182,7 @@ func handleQueryDescribe(c *Core, w *ResponseWriter, r *Request) { return } src := data.NewSource(storage.NewRemoteEngine(), c.root) - info, err := describe.Analyze(r.Context(), req.Query, src, &req.Head) + info, err := describe.Analyze(r.Context(), req.Query, src) if err != nil { w.Error(srverr.ErrInvalid(err)) return diff --git a/service/ztests/curl-delete-where.yaml b/service/ztests/curl-delete-where.yaml index 3380e8cf8c..5cbd08e7e7 100644 --- a/service/ztests/curl-delete-where.yaml +++ b/service/ztests/curl-delete-where.yaml @@ -7,10 +7,10 @@ script: | curl -s -d '{where:"x <= 4"}' $SUPER_DB_LAKE/pool/test/branch/main/delete | sed -E 's/0x[0-9a-f]{40}/xxx/' echo === - super db query -z '*' + super db query -z 'from test' echo === curl -w 'code %{response_code}\n' -d '{where:"x <= 4"}' $SUPER_DB_LAKE/pool/test/branch/main/delete - super db query -z '*' + super db query -z 'from test' inputs: - name: service.sh diff --git a/service/ztests/curl-query-error.yaml b/service/ztests/curl-query-error.yaml index 08ccf79f31..29c61d6a0c 100644 --- a/service/ztests/curl-query-error.yaml +++ b/service/ztests/curl-query-error.yaml @@ -13,11 +13,11 @@ inputs: outputs: - name: stdout data: | - {"type":"Error","kind":"invalid operation","error":"no pool name given"} + {"type":"Error","kind":"invalid operation","error":"query text is missing"} code 400 - {"type":"Error","kind":"invalid operation","error":"no pool name given"} + {"type":"Error","kind":"invalid operation","error":"query text is missing"} code 400 - {"type":"Error","kind":"invalid operation","error":"cannot resolve unknown HEAD at line 1, column 6:\nfrom HEAD\n ~~~~","compilation_errors":[{"Msg":"cannot resolve unknown HEAD","Pos":5,"End":8}]} + {"type":"Error","kind":"invalid operation","error":"HEAD: pool not found at line 1, column 6:\nfrom HEAD\n ~~~~","compilation_errors":[{"Msg":"HEAD: pool not found","Pos":5,"End":8}]} code 400 {"type":"Error","kind":"invalid operation","error":"unknown lake metadata type \"unknownmeta\" in from operator at line 1, column 6:\nfrom :unknownmeta\n ~~~~~~~~~~~~","compilation_errors":[{"Msg":"unknown lake metadata type \"unknownmeta\" in from operator","Pos":5,"End":16}]} code 400 diff --git a/service/ztests/debug.yaml b/service/ztests/debug.yaml index 938747c206..3019f05c0b 100644 --- a/service/ztests/debug.yaml +++ b/service/ztests/debug.yaml @@ -2,7 +2,7 @@ script: | source service.sh super db create -use -q test echo '{x: "foo"}' | super db load -q - - super db query -z 'debug f"debug: {x}"' + super db query -z 'from test | debug f"debug: {x}"' inputs: - name: service.sh diff --git a/service/ztests/delete-where.yaml b/service/ztests/delete-where.yaml index 87f7bbb212..0facc1ca83 100644 --- a/service/ztests/delete-where.yaml +++ b/service/ztests/delete-where.yaml @@ -5,11 +5,11 @@ script: | super db create -use -q -S 1KB -orderby ts:$order test seq 1000 | super -c '{ts:this-1,s:f"val{this-1}"}' - | super db load -q - super db delete -q -where 'ts > 400 and ts <= 500' - super db query -z 'count()' + super db query -z 'from test | count()' super db delete -q -where 's == "val1" or s == "val999"' - super db query -z 'count()' + super db query -z 'from test | count()' ! super db delete -q -where 's == "val1" or s == "val999"' - super db query -z 'count()' + super db query -z 'from test | count()' super db drop -f -q test done diff --git a/service/ztests/delete.yaml b/service/ztests/delete.yaml index f254c6144d..c8a79c1e60 100644 --- a/service/ztests/delete.yaml +++ b/service/ztests/delete.yaml @@ -4,10 +4,10 @@ script: | super db load -q 1.jsup id=$(super db query -f text "from test@main:objects | cut id:=ksuid(id) | tail 1") super db load -q 2.jsup - super db query -z "*" + super db query -z "from test" echo === | tee /dev/stderr super db delete -q $id - super db query -z "*" + super db query -z "from test" echo === | tee /dev/stderr ! super db delete -q 27aaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/service/ztests/query-describe.yaml b/service/ztests/query-describe.yaml index 48afda3425..6a64c9369a 100644 --- a/service/ztests/query-describe.yaml +++ b/service/ztests/query-describe.yaml @@ -20,10 +20,10 @@ inputs: ) | put foo := "bar" - name: agg.spq data: | - count() by key1:=v1, key2 + from test1 | count() by key1:=v1, key2 - name: agg-no-keys.spq data: | - sum(this) + from test1 | sum(this) - name: two-channels.spq data: | fork ( @@ -32,7 +32,7 @@ inputs: ) - name: agg-sort.spq data: | - sum(this) by foo | sort x + from test1 | sum(this) by foo | sort x - name: scope.spq data: | type port = uint16 @@ -50,14 +50,12 @@ outputs: { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": false + "id": "XXX" }, { "kind": "Pool", "name": "test2", - "id": "XXX", - "inferred": false + "id": "XXX" } ], "channels": [ @@ -80,8 +78,7 @@ outputs: "sources": { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": true + "id": "XXX" }, "channels": [ { @@ -103,8 +100,7 @@ outputs: "sources": { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": true + "id": "XXX" }, "channels": [ { @@ -120,14 +116,12 @@ outputs: { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": false + "id": "XXX" }, { "kind": "Pool", "name": "test2", - "id": "XXX", - "inferred": false + "id": "XXX" } ], "channels": [ @@ -159,8 +153,7 @@ outputs: "sources": { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": true + "id": "XXX" }, "channels": [ { @@ -186,8 +179,7 @@ outputs: "sources": { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": false + "id": "XXX" }, "channels": [ { @@ -214,8 +206,7 @@ outputs: "sources": { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": false + "id": "XXX" }, "channels": [ { diff --git a/service/ztests/query-error.yaml b/service/ztests/query-error.yaml index 567e5e61e6..eae67f0e90 100644 --- a/service/ztests/query-error.yaml +++ b/service/ztests/query-error.yaml @@ -12,8 +12,8 @@ inputs: outputs: - name: stderr data: | - status code 400: no pool name given - cannot resolve unknown HEAD at line 1, column 6: + status code 400: query text is missing + HEAD: pool not found at line 1, column 6: from HEAD ~~~~ unknown lake metadata type "unknownmeta" in from operator at line 1, column 6: diff --git a/service/ztests/seek-index-null.yaml b/service/ztests/seek-index-null.yaml index f4230a4a72..1d7cf04d02 100644 --- a/service/ztests/seek-index-null.yaml +++ b/service/ztests/seek-index-null.yaml @@ -4,7 +4,7 @@ script: | echo // $o | tee /dev/stderr super db create -q -seekstride=2KB -orderby ts:$o $o super db load -q -use $o babble.jsup null.jsup - super db query -z -s -use $o "ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" + super db query -z -s "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" done inputs: diff --git a/service/ztests/seek-index.yaml b/service/ztests/seek-index.yaml index 557a1bbdd5..322d16354c 100644 --- a/service/ztests/seek-index.yaml +++ b/service/ztests/seek-index.yaml @@ -4,9 +4,9 @@ script: | echo // $o | tee /dev/stderr super db create -use -seekstride 2KB -orderby ts:$o -q $o super db load -q babble.jsup - source query.sh "ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" - source query.sh "ts == 2020-04-21T23:59:26.06326664Z" - source query.sh "ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar'" + source query.sh "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" + source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z" + source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar'" done inputs: @@ -22,36 +22,36 @@ outputs: - name: stdout data: | // asc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - // ts == 2020-04-21T23:59:26.06326664Z + // from asc | ts == 2020-04-21T23:59:26.06326664Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} // desc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z + // from desc | ts == 2020-04-21T23:59:26.06326664Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - name: stderr data: | // asc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // ts == 2020-04-21T23:59:26.06326664Z + // from asc | ts == 2020-04-21T23:59:26.06326664Z {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} // desc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // ts == 2020-04-21T23:59:26.06326664Z + // from desc | ts == 2020-04-21T23:59:26.06326664Z {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} diff --git a/service/ztests/vacuum.yaml b/service/ztests/vacuum.yaml index 6bfd4c13c5..0cd1837a62 100644 --- a/service/ztests/vacuum.yaml +++ b/service/ztests/vacuum.yaml @@ -7,7 +7,7 @@ script: | super db vacuum -dryrun super db vacuum -f super db revert -q $r - ! super db query -z '*' + ! super db query -z 'from test' inputs: - name: service.sh diff --git a/zfmt/ast.go b/zfmt/ast.go index 5230fee868..42224ef6b2 100644 --- a/zfmt/ast.go +++ b/zfmt/ast.go @@ -605,7 +605,7 @@ func (c *canon) op(p ast.Op) { c.expr(p.Expr, "") } default: - c.open("unknown proc: %T", p) + c.open("unknown operator: %T", p) c.close() } } diff --git a/zfmt/dag.go b/zfmt/dag.go index 5fbe6c4fef..a306207c02 100644 --- a/zfmt/dag.go +++ b/zfmt/dag.go @@ -505,6 +505,9 @@ func (c *canonDAG) op(p dag.Op) { c.expr(p.Filter, "") c.write(")") } + case *dag.NullScan: + c.next() + c.write("null") case *dag.FileScan: c.next() c.write("file %s", p.Path) @@ -548,7 +551,7 @@ func (c *canonDAG) op(p dag.Op) { c.write("output %s", p.Name) default: c.next() - c.open("unknown proc: %T", p) + c.open("unknown operator: %T", p) c.close() } } diff --git a/zfmt/ztests/debug.yaml b/zfmt/ztests/debug.yaml index cb866860f2..a53db1698d 100644 --- a/zfmt/ztests/debug.yaml +++ b/zfmt/ztests/debug.yaml @@ -7,7 +7,7 @@ outputs: data: | debug f"debug: {this}" | head 1 - reader + null | mirror ( => yield "debug: "+cast(this, ) diff --git a/zfmt/ztests/join.yaml b/zfmt/ztests/join.yaml index 71f44d5474..771b74ccd2 100644 --- a/zfmt/ztests/join.yaml +++ b/zfmt/ztests/join.yaml @@ -10,7 +10,7 @@ outputs: from "test.jsup" ) on x=x p:=a === - reader + null | fork ( => pass diff --git a/zfmt/ztests/output.yaml b/zfmt/ztests/output.yaml index 58e77b924d..d97c29dc9b 100644 --- a/zfmt/ztests/output.yaml +++ b/zfmt/ztests/output.yaml @@ -6,7 +6,7 @@ script: | outputs: - name: stdout data: | - reader + null | fork ( => output foo @@ -15,7 +15,7 @@ outputs: | output main ) // === - reader + null | switch x ( case "foo" => output foo diff --git a/zfmt/ztests/over.yaml b/zfmt/ztests/over.yaml index 3814ceb5e2..00b9db16e3 100644 --- a/zfmt/ztests/over.yaml +++ b/zfmt/ztests/over.yaml @@ -42,7 +42,7 @@ outputs: search g ) === - reader + null | yield ( over a | where search("b") diff --git a/zfmt/ztests/precedence-dag.yaml b/zfmt/ztests/precedence-dag.yaml index a6cd7d1a88..807d7e6e0d 100644 --- a/zfmt/ztests/precedence-dag.yaml +++ b/zfmt/ztests/precedence-dag.yaml @@ -16,12 +16,12 @@ inputs: outputs: - name: stdout data: | - reader + null | yield (10+2)/8 | output main - reader + null | where (y==2 or x==4) and z==5 | output main - reader + null | where !(ts<=2) | output main diff --git a/zfmt/ztests/switch.yaml b/zfmt/ztests/switch.yaml index dfb95cb525..00f2510b29 100644 --- a/zfmt/ztests/switch.yaml +++ b/zfmt/ztests/switch.yaml @@ -17,7 +17,7 @@ outputs: tail ) === - reader + null | switch ( case search("a") => head 1 @@ -34,7 +34,7 @@ outputs: tail ) === - reader + null | switch ( case search("a") => head 1 diff --git a/zfmt/ztests/type-value.yaml b/zfmt/ztests/type-value.yaml index 5b9f54e4e9..86921bd3c7 100644 --- a/zfmt/ztests/type-value.yaml +++ b/zfmt/ztests/type-value.yaml @@ -6,6 +6,6 @@ outputs: - name: stdout data: | search is() and bar - reader + null | where is(<(uint16,ip)>) and search(80) | output main diff --git a/zfmt/ztests/yield-shortcut.yaml b/zfmt/ztests/yield-shortcut.yaml index a1e9447279..d6c9e21a85 100644 --- a/zfmt/ztests/yield-shortcut.yaml +++ b/zfmt/ztests/yield-shortcut.yaml @@ -20,18 +20,18 @@ outputs: yield |{"foo":1,"bar":2}| yield <(int64,string)>(1) === - reader + null | yield {x:1,...y} | output main - reader + null | yield [1,2,3] | output main - reader + null | yield |["foo","bar"]| | output main - reader + null | yield |{"foo":1,"bar":2}| | output main - reader + null | yield cast(1, <(int64,string)>) | output main