From d483049ea64fdf5421e3d5d6923c8698c488b29d Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Fri, 1 Nov 2024 09:34:48 -0700 Subject: [PATCH 1/2] align from-less query semantics with SQL This commit changes the interpretation of a query that omits the from operator. Previously, a lake query would use the default "HEAD" parameter as an implied "from HEAD". This contradicts SQL which assumes a table with one empty row (like the implied yield null case). The new assumption in a lake query for a from-less query is an implied null source so that a "select 'hello'" query will return not a single 'hello' rather than a sequence based on the implied read from HEAD. Similary, a from-less command-line query (i.e., super -c) with no file arguments also has an automatic null scan inserted. As part of this change, we moved the logic for inserting the null scan into the DAG (along with the logic for inserting DefaultScan of zio.Readers) into the semantic pass. These changes mean that lake queries that read from a pool require an explicit "from" and there is no impleied pool from HEAD. Because of this, we removed the HEAD references in the code paths that involve lake queries. HEAD is still used by the various "super db" commands to refer to a default pool/branch/commit. Currently, the null scan does not follow the done protocol and restart. We will need to add this when we want from-less subqueries to restart. --- api/api.go | 4 +- api/client/connection.go | 6 +- cmd/super/compile/shared.go | 4 +- cmd/super/db/branch/command.go | 2 +- cmd/super/db/log/command.go | 2 +- cmd/super/db/ls/command.go | 2 +- cmd/super/db/query/command.go | 6 +- cmd/super/internal/lakemanage/scan.go | 2 +- compiler/ast/ast.go | 6 +- compiler/dag/op.go | 4 ++ compiler/dag/unpack.go | 1 + compiler/describe/analyze.go | 62 +++++++++--------- compiler/file.go | 4 +- compiler/job.go | 29 +++++---- compiler/kernel/op.go | 5 +- compiler/lake.go | 12 ++-- compiler/parser/api.go | 13 +++- compiler/reader.go | 6 +- compiler/semantic/analyzer.go | 76 +++++++---------------- compiler/semantic/op.go | 20 ++---- compiler/srcfiles/list.go | 3 - compiler/ztests/head.yaml | 4 +- compiler/ztests/merge-filters.yaml | 8 +-- compiler/ztests/remove-passops.yaml | 4 +- compiler/ztests/tail.yaml | 4 +- compiler/ztests/udf-implied-where.yaml | 2 +- docs/libraries/go.md | 2 +- fuzz/fuzz.go | 2 +- lake/api/api.go | 13 ++-- lake/api/local.go | 4 +- lake/api/remote.go | 4 +- lake/ztests/checkout-dash-p.yaml | 3 +- lake/ztests/debug.yaml | 2 +- lake/ztests/delete-where-missing.yaml | 4 +- lake/ztests/delete-where.yaml | 4 +- lake/ztests/delete.yaml | 4 +- lake/ztests/query-error.yaml | 4 +- lake/ztests/revert-revert.yaml | 4 +- lake/ztests/revert.yaml | 6 +- lake/ztests/seek-index-boundaries.yaml | 55 ++++++++-------- lake/ztests/seek-index-null.yaml | 2 +- lake/ztests/seek-index-ts.yaml | 35 +++++------ lake/ztests/vacuum.yaml | 2 +- runtime/compiler.go | 6 +- runtime/sam/expr/filter_test.go | 3 +- runtime/sam/op/meta/ztests/nulls-max.yaml | 2 +- service/auth_test.go | 4 +- service/client_test.go | 4 +- service/handlers.go | 4 +- service/ztests/curl-delete-where.yaml | 4 +- service/ztests/curl-query-error.yaml | 6 +- service/ztests/debug.yaml | 2 +- service/ztests/delete-where.yaml | 6 +- service/ztests/delete.yaml | 4 +- service/ztests/query-describe.yaml | 33 ++++------ service/ztests/query-error.yaml | 4 +- service/ztests/seek-index-null.yaml | 2 +- service/ztests/seek-index.yaml | 30 ++++----- service/ztests/vacuum.yaml | 2 +- zfmt/ast.go | 2 +- zfmt/dag.go | 5 +- zfmt/ztests/debug.yaml | 2 +- zfmt/ztests/join.yaml | 2 +- zfmt/ztests/output.yaml | 4 +- zfmt/ztests/over.yaml | 2 +- zfmt/ztests/precedence-dag.yaml | 6 +- zfmt/ztests/switch.yaml | 4 +- zfmt/ztests/type-value.yaml | 2 +- zfmt/ztests/yield-shortcut.yaml | 10 +-- 69 files changed, 272 insertions(+), 324 deletions(-) diff --git a/api/api.go b/api/api.go index 4e64653da0..76343cf7f3 100644 --- a/api/api.go +++ b/api/api.go @@ -4,7 +4,6 @@ import ( "context" "github.com/brimdata/super/compiler/srcfiles" - "github.com/brimdata/super/lakeparse" "github.com/brimdata/super/order" "github.com/brimdata/super/pkg/field" "github.com/brimdata/super/pkg/nano" @@ -98,8 +97,7 @@ type EventBranch struct { } type QueryRequest struct { - Query string `json:"query"` - Head lakeparse.Commitish `json:"head"` + Query string `json:"query"` } type QueryChannelSet struct { diff --git a/api/client/connection.go b/api/client/connection.go index 0a0db7bfec..30285d16a5 100644 --- a/api/client/connection.go +++ b/api/client/connection.go @@ -19,7 +19,6 @@ import ( "github.com/brimdata/super/compiler/srcfiles" "github.com/brimdata/super/lake" "github.com/brimdata/super/lake/branches" - "github.com/brimdata/super/lakeparse" "github.com/brimdata/super/runtime/exec" "github.com/brimdata/super/zio/zngio" "github.com/brimdata/super/zson" @@ -296,15 +295,12 @@ func (c *Connection) Revert(ctx context.Context, poolID ksuid.KSUID, branchName // // As for Connection.Do, if the returned error is nil, the user is expected to // call Response.Body.Close. -func (c *Connection) Query(ctx context.Context, head *lakeparse.Commitish, src string, filenames ...string) (*Response, error) { +func (c *Connection) Query(ctx context.Context, src string, filenames ...string) (*Response, error) { files, err := srcfiles.Concat(filenames, src) if err != nil { return nil, err } body := api.QueryRequest{Query: string(files.Text)} - if head != nil { - body.Head = *head - } req := c.NewRequest(ctx, http.MethodPost, "/query?ctrl=T", body) res, err := c.Do(req) if ae := (*api.Error)(nil); errors.As(err, &ae) && len(ae.CompilationErrors) > 0 { diff --git a/cmd/super/compile/shared.go b/cmd/super/compile/shared.go index 75113d7745..aefec27259 100644 --- a/cmd/super/compile/shared.go +++ b/cmd/super/compile/shared.go @@ -76,12 +76,12 @@ func (s *Shared) Run(ctx context.Context, args []string, lakeFlags *lakeflags.Fl } return s.writeValue(ctx, ast.Parsed()) } - runtime, err := compiler.NewJob(runtime.DefaultContext(), ast, data.NewSource(nil, lk), nil) + runtime, err := compiler.NewJob(runtime.DefaultContext(), ast, data.NewSource(nil, lk), false) if err != nil { return err } if desc { - description, err := describe.AnalyzeDAG(ctx, runtime.Entry(), data.NewSource(nil, lk), nil) + description, err := describe.AnalyzeDAG(ctx, runtime.Entry(), data.NewSource(nil, lk)) if err != nil { return err } diff --git a/cmd/super/db/branch/command.go b/cmd/super/db/branch/command.go index 6d3c70f9df..2766c4a87a 100644 --- a/cmd/super/db/branch/command.go +++ b/cmd/super/db/branch/command.go @@ -131,7 +131,7 @@ func (c *Command) list(ctx context.Context, lake api.Interface) error { if err != nil { return err } - q, err := lake.Query(ctx, nil, query) + q, err := lake.Query(ctx, query) if err != nil { w.Close() return err diff --git a/cmd/super/db/log/command.go b/cmd/super/db/log/command.go index 5536483756..de2247884b 100644 --- a/cmd/super/db/log/command.go +++ b/cmd/super/db/log/command.go @@ -72,7 +72,7 @@ func (c *Command) Run(args []string) error { return err } defer w.Close() - q, err := lake.Query(ctx, nil, query) + q, err := lake.Query(ctx, query) if err != nil { return err } diff --git a/cmd/super/db/ls/command.go b/cmd/super/db/ls/command.go index bd41be1272..82d1723137 100644 --- a/cmd/super/db/ls/command.go +++ b/cmd/super/db/ls/command.go @@ -88,7 +88,7 @@ func (c *Command) Run(args []string) error { if err != nil { return err } - q, err := lake.Query(ctx, nil, query) + q, err := lake.Query(ctx, query) if err != nil { w.Close() return err diff --git a/cmd/super/db/query/command.go b/cmd/super/db/query/command.go index 2e694e7a44..79343b881e 100644 --- a/cmd/super/db/query/command.go +++ b/cmd/super/db/query/command.go @@ -5,7 +5,6 @@ import ( "os" "github.com/brimdata/super/cli/outputflags" - "github.com/brimdata/super/cli/poolflags" "github.com/brimdata/super/cli/queryflags" "github.com/brimdata/super/cli/runtimeflags" "github.com/brimdata/super/cmd/super/db" @@ -33,7 +32,6 @@ func init() { type Command struct { *db.Command outputFlags outputflags.Flags - poolFlags poolflags.Flags queryFlags queryflags.Flags runtimeFlags runtimeflags.Flags } @@ -41,7 +39,6 @@ type Command struct { func New(parent charm.Command, f *flag.FlagSet) (charm.Command, error) { c := &Command{Command: parent.(*db.Command)} c.outputFlags.SetFlags(f) - c.poolFlags.SetFlags(f) c.queryFlags.SetFlags(f) c.runtimeFlags.SetFlags(f) return c, nil @@ -68,8 +65,7 @@ func (c *Command) Run(args []string) error { if err != nil { return err } - head, _ := c.poolFlags.HEAD() - query, err := lake.Query(ctx, head, src, c.queryFlags.Includes...) + query, err := lake.Query(ctx, src, c.queryFlags.Includes...) if err != nil { w.Close() return err diff --git a/cmd/super/internal/lakemanage/scan.go b/cmd/super/internal/lakemanage/scan.go index e4ccb87bed..14cfd1f745 100644 --- a/cmd/super/internal/lakemanage/scan.go +++ b/cmd/super/internal/lakemanage/scan.go @@ -74,7 +74,7 @@ type objectIterator struct { func newObjectIterator(ctx context.Context, lake api.Interface, head *lakeparse.Commitish) (*objectIterator, error) { query := fmt.Sprintf(iteratorQuery, head.Pool, head.Branch, head.Pool, head.Branch) - q, err := lake.Query(ctx, nil, query) + q, err := lake.Query(ctx, query) if err != nil { return nil, err } diff --git a/compiler/ast/ast.go b/compiler/ast/ast.go index 34a4f2fbe0..ad2bbab0d0 100644 --- a/compiler/ast/ast.go +++ b/compiler/ast/ast.go @@ -574,8 +574,10 @@ type ( Loc `json:"loc"` } Delete struct { - Kind string `json:"kind" unpack:""` - Loc `json:"loc"` // dummy field, not needed except to implement Node + Kind string `json:"kind" unpack:""` + Pool string `json:"pool"` + Branch string `json:"branch"` + Loc `json:"loc"` // dummy field, not needed except to implement Node } ) diff --git a/compiler/dag/op.go b/compiler/dag/op.go index 7d31bdb685..623d6ac302 100644 --- a/compiler/dag/op.go +++ b/compiler/dag/op.go @@ -246,6 +246,9 @@ type ( Kind string `json:"kind" unpack:""` Meta string `json:"meta"` } + NullScan struct { + Kind string `json:"kind" unpack:""` + } ) var LakeMetas = map[string]struct{}{ @@ -273,6 +276,7 @@ func (*DeleteScan) OpNode() {} func (*LakeMetaScan) OpNode() {} func (*PoolMetaScan) OpNode() {} func (*CommitMetaScan) OpNode() {} +func (*NullScan) OpNode() {} func (*Lister) OpNode() {} func (*Slicer) OpNode() {} diff --git a/compiler/dag/unpack.go b/compiler/dag/unpack.go index 76d105da6f..572e49fcb1 100644 --- a/compiler/dag/unpack.go +++ b/compiler/dag/unpack.go @@ -42,6 +42,7 @@ var unpacker = unpack.New( MapExpr{}, Merge{}, Mirror{}, + NullScan{}, Output{}, Over{}, OverExpr{}, diff --git a/compiler/describe/analyze.go b/compiler/describe/analyze.go index 61e65dec08..8845e11997 100644 --- a/compiler/describe/analyze.go +++ b/compiler/describe/analyze.go @@ -10,7 +10,6 @@ import ( "github.com/brimdata/super/compiler/parser" "github.com/brimdata/super/compiler/semantic" "github.com/brimdata/super/lake" - "github.com/brimdata/super/lakeparse" "github.com/brimdata/super/order" "github.com/brimdata/super/pkg/field" "github.com/segmentio/ksuid" @@ -31,21 +30,23 @@ type ( Meta string `json:"meta"` } Pool struct { - Kind string `json:"kind"` - Name string `json:"name"` - ID ksuid.KSUID `json:"id"` - Inferred bool `json:"inferred"` + Kind string `json:"kind"` + Name string `json:"name"` + ID ksuid.KSUID `json:"id"` } Path struct { - Kind string `json:"kind"` - URI string `json:"uri"` - Inferred bool `json:"inferred"` + Kind string `json:"kind"` + URI string `json:"uri"` + } + Null struct { + Kind string `json:"kind"` } ) func (*LakeMeta) Source() {} func (*Pool) Source() {} func (*Path) Source() {} +func (*Null) Source() {} type Channel struct { Name string `json:"name"` @@ -53,26 +54,22 @@ type Channel struct { Sort order.SortKeys `json:"sort"` } -func Analyze(ctx context.Context, query string, src *data.Source, head *lakeparse.Commitish) (*Info, error) { +func Analyze(ctx context.Context, query string, src *data.Source) (*Info, error) { ast, err := parser.ParseQuery(query) if err != nil { return nil, err } - entry, err := semantic.Analyze(ctx, ast, src, head) + entry, err := semantic.Analyze(ctx, ast, src, false) if err != nil { return nil, err } - return AnalyzeDAG(ctx, entry, src, head) + return AnalyzeDAG(ctx, entry, src) } -func AnalyzeDAG(ctx context.Context, entry dag.Seq, src *data.Source, head *lakeparse.Commitish) (*Info, error) { - srcInferred := !semantic.HasSource(entry) - if err := semantic.AddDefaultSource(ctx, &entry, src, head); err != nil { - return nil, err - } +func AnalyzeDAG(ctx context.Context, entry dag.Seq, src *data.Source) (*Info, error) { var err error var info Info - if info.Sources, err = describeSources(ctx, src.Lake(), entry[0], srcInferred); err != nil { + if info.Sources, err = describeSources(ctx, src.Lake(), entry[0]); err != nil { return nil, err } sortKeys, err := optimizer.New(ctx, src).SortKeys(entry) @@ -102,14 +99,14 @@ func AnalyzeDAG(ctx context.Context, entry dag.Seq, src *data.Source, head *lake return &info, nil } -func describeSources(ctx context.Context, lk *lake.Root, o dag.Op, inferred bool) ([]Source, error) { +func describeSources(ctx context.Context, lk *lake.Root, o dag.Op) ([]Source, error) { switch o := o.(type) { case *dag.Scope: - return describeSources(ctx, lk, o.Body[0], inferred) + return describeSources(ctx, lk, o.Body[0]) case *dag.Fork: var s []Source for _, p := range o.Paths { - out, err := describeSources(ctx, lk, p[0], inferred) + out, err := describeSources(ctx, lk, p[0]) if err != nil { return nil, err } @@ -117,19 +114,21 @@ func describeSources(ctx context.Context, lk *lake.Root, o dag.Op, inferred bool } return s, nil case *dag.DefaultScan: - return []Source{&Path{Kind: "Path", URI: "stdio://stdin", Inferred: inferred}}, nil + return []Source{&Path{Kind: "Path", URI: "stdio://stdin"}}, nil + case *dag.NullScan: + return []Source{&Null{Kind: "Null"}}, nil case *dag.FileScan: - return []Source{&Path{Kind: "Path", URI: o.Path, Inferred: inferred}}, nil + return []Source{&Path{Kind: "Path", URI: o.Path}}, nil case *dag.HTTPScan: - return []Source{&Path{Kind: "Path", URI: o.URL, Inferred: inferred}}, nil + return []Source{&Path{Kind: "Path", URI: o.URL}}, nil case *dag.PoolScan: - return sourceOfPool(ctx, lk, o.ID, inferred) + return sourceOfPool(ctx, lk, o.ID) case *dag.Lister: - return sourceOfPool(ctx, lk, o.Pool, inferred) + return sourceOfPool(ctx, lk, o.Pool) case *dag.SeqScan: - return sourceOfPool(ctx, lk, o.Pool, inferred) + return sourceOfPool(ctx, lk, o.Pool) case *dag.CommitMetaScan: - return sourceOfPool(ctx, lk, o.Pool, inferred) + return sourceOfPool(ctx, lk, o.Pool) case *dag.LakeMetaScan: return []Source{&LakeMeta{Kind: "LakeMeta", Meta: o.Meta}}, nil default: @@ -137,16 +136,15 @@ func describeSources(ctx context.Context, lk *lake.Root, o dag.Op, inferred bool } } -func sourceOfPool(ctx context.Context, lk *lake.Root, id ksuid.KSUID, inferred bool) ([]Source, error) { +func sourceOfPool(ctx context.Context, lk *lake.Root, id ksuid.KSUID) ([]Source, error) { p, err := lk.OpenPool(ctx, id) if err != nil { return nil, err } return []Source{&Pool{ - Kind: "Pool", - ID: id, - Name: p.Name, - Inferred: inferred, + Kind: "Pool", + ID: id, + Name: p.Name, }}, nil } diff --git a/compiler/file.go b/compiler/file.go index 3c80c08bab..8182e7ddc6 100644 --- a/compiler/file.go +++ b/compiler/file.go @@ -23,7 +23,7 @@ func NewFileSystemCompiler(engine storage.Engine) runtime.Compiler { } func (f *fsCompiler) NewQuery(rctx *runtime.Context, ast *parser.AST, readers []zio.Reader) (runtime.Query, error) { - job, err := NewJob(rctx, ast, f.src, nil) + job, err := NewJob(rctx, ast, f.src, len(readers) > 0) if err != nil { return nil, err } @@ -45,7 +45,7 @@ func (f *fsCompiler) NewQuery(rctx *runtime.Context, ast *parser.AST, readers [] return optimizeAndBuild(job, readers) } -func (*fsCompiler) NewLakeQuery(_ *runtime.Context, ast *parser.AST, parallelism int, head *lakeparse.Commitish) (runtime.Query, error) { +func (*fsCompiler) NewLakeQuery(_ *runtime.Context, ast *parser.AST, parallelism int) (runtime.Query, error) { panic("NewLakeQuery called on compiler.fsCompiler") } diff --git a/compiler/job.go b/compiler/job.go index 15b74f291b..3e7b06c41f 100644 --- a/compiler/job.go +++ b/compiler/job.go @@ -2,6 +2,7 @@ package compiler import ( "errors" + "fmt" "github.com/brimdata/super/compiler/dag" "github.com/brimdata/super/compiler/data" @@ -28,7 +29,7 @@ type Job struct { entry dag.Seq } -func NewJob(rctx *runtime.Context, ast *parser.AST, src *data.Source, head *lakeparse.Commitish) (*Job, error) { +func NewJob(rctx *runtime.Context, ast *parser.AST, src *data.Source, extInput bool) (*Job, error) { // An AST always begins with a Sequential op with at least one // operator. If the first op is a From or a Parallel whose branches // are Sequentials with a leading From, then we presume there is @@ -39,10 +40,7 @@ func NewJob(rctx *runtime.Context, ast *parser.AST, src *data.Source, head *lake // caller via runtime.readers. In most cases, the AST is left // with an ast.From at the entry point, and hence a dag.From for the // DAG's entry point. - if len(ast.Parsed()) == 0 { - return nil, errors.New("internal error: AST seq cannot be empty") - } - entry, err := semantic.AnalyzeAddSource(rctx.Context, ast, src, head) + entry, err := semantic.Analyze(rctx.Context, ast, src, extInput) if err != nil { return nil, err } @@ -131,10 +129,14 @@ func VectorCompile(rctx *runtime.Context, query string, object *vcache.Object) ( return nil, err } src := &data.Source{} - entry, err := semantic.Analyze(rctx.Context, ast, src, nil) + entry, err := semantic.Analyze(rctx.Context, ast, src, true) if err != nil { return nil, err } + if _, ok := entry[0].(*dag.DefaultScan); !ok { + panic("DAG assumptions violated") + } + entry = entry[1:] puller := vam.NewVectorProjection(rctx.Zctx, object, nil) //XXX project all builder := kernel.NewBuilder(rctx, src) outputs, err := builder.BuildWithPuller(entry, puller) @@ -153,7 +155,7 @@ func VectorFilterCompile(rctx *runtime.Context, query string, src *data.Source, // Eventually the semantic analyzer + kernel will resolve the pool but // for now just do this manually. if !src.IsLake() { - return nil, errors.New("non-lake searches not supported") + return nil, errors.New("non-lake vectorized search not supported") } poolID, err := src.PoolID(rctx.Context, head.Pool) if err != nil { @@ -163,18 +165,23 @@ func VectorFilterCompile(rctx *runtime.Context, query string, src *data.Source, if err != nil { return nil, err } - ast, err := parser.ParseQuery(query) + spec, err := head.FromSpec("") + if err != nil { + return nil, err + } + ast, err := parser.ParseQuery(fmt.Sprintf("%s | %s", spec, query)) if err != nil { return nil, err } - entry, err := semantic.Analyze(rctx.Context, ast, src, nil) + entry, err := semantic.Analyze(rctx.Context, ast, src, false) if err != nil { return nil, err } - if len(entry) != 2 { + // from -> filter -> output + if len(entry) != 3 { return nil, errors.New("filter query must have a single op") } - f, ok := entry[0].(*dag.Filter) + f, ok := entry[1].(*dag.Filter) if !ok { return nil, errors.New("filter query must be a single filter op") } diff --git a/compiler/kernel/op.go b/compiler/kernel/op.go index 876556d838..f39793445e 100644 --- a/compiler/kernel/op.go +++ b/compiler/kernel/op.go @@ -287,6 +287,9 @@ func (b *Builder) compileLeaf(o dag.Op, parent zbuf.Puller) (zbuf.Puller, error) scanners = append(scanners, scanner) } return zbuf.MultiScanner(scanners...), nil + case *dag.NullScan: + //XXX we need something that implements the done protocol and restarst + return zbuf.NewPuller(zbuf.NewArray([]super.Value{super.Null})), nil case *dag.Lister: if parent != nil { return nil, errors.New("internal error: data source cannot have a parent operator") @@ -744,7 +747,7 @@ func isEntry(seq dag.Seq) bool { return false } switch op := seq[0].(type) { - case *dag.Lister, *dag.DefaultScan, *dag.FileScan, *dag.HTTPScan, *dag.PoolScan, *dag.LakeMetaScan, *dag.PoolMetaScan, *dag.CommitMetaScan: + case *dag.Lister, *dag.DefaultScan, *dag.FileScan, *dag.HTTPScan, *dag.PoolScan, *dag.LakeMetaScan, *dag.PoolMetaScan, *dag.CommitMetaScan, *dag.NullScan: return true case *dag.Scope: return isEntry(op.Body) diff --git a/compiler/lake.go b/compiler/lake.go index 1287f88e0c..3f4694e8e3 100644 --- a/compiler/lake.go +++ b/compiler/lake.go @@ -31,8 +31,8 @@ func NewLakeCompiler(r *lake.Root) runtime.Compiler { return &lakeCompiler{src: data.NewSource(storage.NewRemoteEngine(), r)} } -func (l *lakeCompiler) NewLakeQuery(rctx *runtime.Context, ast *parser.AST, parallelism int, head *lakeparse.Commitish) (runtime.Query, error) { - job, err := NewJob(rctx, ast, l.src, head) +func (l *lakeCompiler) NewLakeQuery(rctx *runtime.Context, ast *parser.AST, parallelism int) (runtime.Query, error) { + job, err := NewJob(rctx, ast, l.src, false) if err != nil { return nil, err } @@ -57,7 +57,7 @@ func (l *lakeCompiler) NewLakeQuery(rctx *runtime.Context, ast *parser.AST, para } func (l *lakeCompiler) NewLakeDeleteQuery(rctx *runtime.Context, ast *parser.AST, head *lakeparse.Commitish) (runtime.DeleteQuery, error) { - job, err := newDeleteJob(rctx, ast, l.src, head) + job, err := newDeleteJob(rctx, ast, head.Pool, head.Branch, l.src) if err != nil { return nil, err } @@ -76,15 +76,15 @@ func (InvalidDeleteWhereQuery) Error() string { return "invalid delete where query: must be a single filter operation" } -func newDeleteJob(rctx *runtime.Context, parsed *parser.AST, src *data.Source, head *lakeparse.Commitish) (*Job, error) { - if err := parsed.ConvertToDeleteWhere(); err != nil { +func newDeleteJob(rctx *runtime.Context, parsed *parser.AST, pool, branch string, src *data.Source) (*Job, error) { + if err := parsed.ConvertToDeleteWhere(pool, branch); err != nil { return nil, err } seq := parsed.Parsed() if len(seq) != 2 { return nil, &InvalidDeleteWhereQuery{} } - entry, err := semantic.Analyze(rctx.Context, parsed, src, head) + entry, err := semantic.Analyze(rctx.Context, parsed, src, false) if err != nil { return nil, err } diff --git a/compiler/parser/api.go b/compiler/parser/api.go index 72c1dcf0e1..6dff677816 100644 --- a/compiler/parser/api.go +++ b/compiler/parser/api.go @@ -24,11 +24,15 @@ func (a *AST) Files() *srcfiles.List { return a.files } -func (a *AST) ConvertToDeleteWhere() error { +func (a *AST) ConvertToDeleteWhere(pool, branch string) error { if len(a.seq) == 0 { - return errors.New("internal error: AST seq cannot be empty") + return errors.New("delete where command requires an expression") } - a.seq.Prepend(&ast.Delete{Kind: "Delete"}) + a.seq.Prepend(&ast.Delete{ + Kind: "Delete", + Pool: pool, + Branch: branch, + }) return nil } @@ -39,6 +43,9 @@ func ParseQuery(query string, filenames ...string) (*AST, error) { if err != nil { return nil, err } + if files.Text == "" { + return &AST{seq: []ast.Op{}, files: files}, nil + } p, err := Parse("", []byte(files.Text), Recover(false)) if err != nil { if err := convertParseErrs(err, files); err != nil { diff --git a/compiler/reader.go b/compiler/reader.go index c601bc6240..a308ae8529 100644 --- a/compiler/reader.go +++ b/compiler/reader.go @@ -21,7 +21,7 @@ func (i *anyCompiler) NewQuery(rctx *runtime.Context, ast *parser.AST, readers [ if len(readers) != 1 { return nil, fmt.Errorf("NewQuery: Zed program expected %d readers", len(readers)) } - job, err := NewJob(rctx, ast, data.NewSource(nil, nil), nil) + job, err := NewJob(rctx, ast, data.NewSource(nil, nil), true) if err != nil { return nil, err } @@ -30,7 +30,7 @@ func (i *anyCompiler) NewQuery(rctx *runtime.Context, ast *parser.AST, readers [ // XXX currently used only by group-by test, need to deprecate func CompileWithSortKey(rctx *runtime.Context, ast *parser.AST, r zio.Reader, sortKey order.SortKey) (*exec.Query, error) { - job, err := NewJob(rctx, ast, data.NewSource(nil, nil), nil) + job, err := NewJob(rctx, ast, data.NewSource(nil, nil), true) if err != nil { return nil, err } @@ -42,7 +42,7 @@ func CompileWithSortKey(rctx *runtime.Context, ast *parser.AST, r zio.Reader, so return optimizeAndBuild(job, []zio.Reader{r}) } -func (*anyCompiler) NewLakeQuery(rctx *runtime.Context, ast *parser.AST, parallelism int, head *lakeparse.Commitish) (runtime.Query, error) { +func (*anyCompiler) NewLakeQuery(rctx *runtime.Context, ast *parser.AST, parallelism int) (runtime.Query, error) { panic("NewLakeQuery called on compiler.anyCompiler") } diff --git a/compiler/semantic/analyzer.go b/compiler/semantic/analyzer.go index 9a2752b239..47c2975810 100644 --- a/compiler/semantic/analyzer.go +++ b/compiler/semantic/analyzer.go @@ -10,42 +10,36 @@ import ( "github.com/brimdata/super/compiler/data" "github.com/brimdata/super/compiler/parser" "github.com/brimdata/super/compiler/srcfiles" - "github.com/brimdata/super/lakeparse" ) // Analyze performs a semantic analysis of the AST, translating it from AST // to DAG form, resolving syntax ambiguities, and performing constant propagation. // After semantic analysis, the DAG is ready for either optimization or compilation. -func Analyze(ctx context.Context, ast *parser.AST, source *data.Source, head *lakeparse.Commitish) (dag.Seq, error) { +func Analyze(ctx context.Context, ast *parser.AST, source *data.Source, extInput bool) (dag.Seq, error) { files := ast.Files() - a := newAnalyzer(ctx, files, source, head) - s := a.semSeq(ast.Parsed()) - s = a.checkOutputs(true, s) - return s, files.Error() -} - -// AnalyzeAddSource is the same as Analyze but it adds a default source if the -// DAG does not have one. -func AnalyzeAddSource(ctx context.Context, ast *parser.AST, source *data.Source, head *lakeparse.Commitish) (dag.Seq, error) { - files := ast.Files() - a := newAnalyzer(ctx, files, source, head) - s := a.semSeq(ast.Parsed()) - s = a.checkOutputs(true, s) - if err := files.Error(); err != nil { - return nil, err - } - if !HasSource(s) { - if err := AddDefaultSource(ctx, &s, source, head); err != nil { - return nil, err + a := newAnalyzer(ctx, files, source) + seq := a.semSeq(ast.Parsed()) + if !HasSource(seq) { + if a.source.IsLake() { + if len(seq) == 0 { + return nil, errors.New("query text is missing") + } + seq.Prepend(&dag.NullScan{Kind: "NullScan"}) + } else if extInput { + seq.Prepend(&dag.DefaultScan{Kind: "DefaultScan"}) + } else { + // This is a local query and there's not external input + // (i.e., no command-line file args) + seq.Prepend(&dag.NullScan{Kind: "NullScan"}) } } - return s, nil + seq = a.checkOutputs(true, seq) + return seq, files.Error() } type analyzer struct { ctx context.Context files *srcfiles.List - head *lakeparse.Commitish opStack []*ast.OpDecl outputs map[*dag.Output]ast.Node source *data.Source @@ -53,11 +47,10 @@ type analyzer struct { zctx *super.Context } -func newAnalyzer(ctx context.Context, files *srcfiles.List, source *data.Source, head *lakeparse.Commitish) *analyzer { +func newAnalyzer(ctx context.Context, files *srcfiles.List, source *data.Source) *analyzer { return &analyzer{ ctx: ctx, files: files, - head: head, outputs: make(map[*dag.Output]ast.Node), source: source, scope: NewScope(nil), @@ -66,8 +59,11 @@ func newAnalyzer(ctx context.Context, files *srcfiles.List, source *data.Source, } func HasSource(seq dag.Seq) bool { + if len(seq) == 0 { + return false + } switch op := seq[0].(type) { - case *dag.FileScan, *dag.HTTPScan, *dag.PoolScan, *dag.LakeMetaScan, *dag.PoolMetaScan, *dag.CommitMetaScan, *dag.DeleteScan: + case *dag.FileScan, *dag.HTTPScan, *dag.PoolScan, *dag.LakeMetaScan, *dag.PoolMetaScan, *dag.CommitMetaScan, *dag.DeleteScan, *dag.NullScan: return true case *dag.Fork: return HasSource(op.Paths[0]) @@ -77,34 +73,6 @@ func HasSource(seq dag.Seq) bool { return false } -func AddDefaultSource(ctx context.Context, seq *dag.Seq, source *data.Source, head *lakeparse.Commitish) error { - if HasSource(*seq) { - return nil - } - // No from so add a source. - if head == nil { - seq.Prepend(&dag.DefaultScan{Kind: "DefaultScan"}) - return nil - } - // Verify pool exists for HEAD - if _, err := source.PoolID(ctx, head.Pool); err != nil { - return err - } - fromHead := &ast.From{ - Kind: "From", - Elems: []*ast.FromElem{{ - Kind: "FromElem", - Entity: &ast.Name{ - Kind: "Name", - Text: "HEAD", - }, - }}, - } - headSeq := newAnalyzer(ctx, &srcfiles.List{}, source, head).semFrom(fromHead, nil) - seq.Prepend(headSeq[0]) - return nil -} - func (a *analyzer) enterScope() { a.scope = NewScope(a.scope) } diff --git a/compiler/semantic/op.go b/compiler/semantic/op.go index 9bdd53f44a..77d8a87ce8 100644 --- a/compiler/semantic/op.go +++ b/compiler/semantic/op.go @@ -339,14 +339,6 @@ func (a *analyzer) semPool(nameLoc ast.Node, poolName string, args *ast.PoolArgs meta = nullableName(args.Meta) tap = args.Tap } - if poolName == "HEAD" { - if a.head == nil || a.head.Pool == "" { - a.error(nameLoc, errors.New("cannot resolve unknown HEAD")) - return badOp() - } - poolName = a.head.Pool - commit = a.head.Branch - } poolID, err := a.source.PoolID(a.ctx, poolName) if err != nil { a.error(nameLoc, err) @@ -422,20 +414,16 @@ func (a *analyzer) semDelete(op *ast.Delete) dag.Op { a.error(op, errors.New("deletion requires data lake")) return badOp() } - // delete-where only supports deleting at head - pool := a.head.Pool - commit := a.head.Branch - - poolID, err := a.source.PoolID(a.ctx, pool) + poolID, err := a.source.PoolID(a.ctx, op.Pool) if err != nil { a.error(op, err) return badOp() } var commitID ksuid.KSUID - if commit != "" { + if op.Branch != "" { var err error - if commitID, err = lakeparse.ParseID(commit); err != nil { - commitID, err = a.source.CommitObject(a.ctx, poolID, commit) + if commitID, err = lakeparse.ParseID(op.Branch); err != nil { + commitID, err = a.source.CommitObject(a.ctx, poolID, op.Branch) if err != nil { a.error(op, err) return badOp() diff --git a/compiler/srcfiles/list.go b/compiler/srcfiles/list.go index 069ffb4bf6..0f95f5e09e 100644 --- a/compiler/srcfiles/list.go +++ b/compiler/srcfiles/list.go @@ -42,9 +42,6 @@ func Concat(filenames []string, query string) (*List, error) { b.Write(bb) b.WriteByte('\n') } - if b.Len() == 0 && query == "" { - query = "*" - } // Empty string is the unnamed query text while the included files all // have names. files = append(files, newFile("", b.Len(), []byte(query))) diff --git a/compiler/ztests/head.yaml b/compiler/ztests/head.yaml index 4bb728d6cd..38f8f1d536 100644 --- a/compiler/ztests/head.yaml +++ b/compiler/ztests/head.yaml @@ -9,11 +9,11 @@ script: | outputs: - name: stdout data: | - reader + null | head 1 | output main === - reader + null | ( const x = 1 diff --git a/compiler/ztests/merge-filters.yaml b/compiler/ztests/merge-filters.yaml index c9a62b4f49..609bb3c533 100644 --- a/compiler/ztests/merge-filters.yaml +++ b/compiler/ztests/merge-filters.yaml @@ -1,5 +1,5 @@ script: | - super compile -C -O 'where a | where b' + super compile -C -O 'from foo | where a | where b' echo === super compile -C -O 'from ( file a => where b | where c file d => where e | where f ) | where g' echo === @@ -10,7 +10,7 @@ script: | outputs: - name: stdout data: | - reader filter (a and b) + file foo filter (a and b) | output main === fork ( @@ -21,13 +21,13 @@ outputs: ) | output main === - reader + null | over a => ( where b and c ) | output main === - reader + null | fork ( => where a and b diff --git a/compiler/ztests/remove-passops.yaml b/compiler/ztests/remove-passops.yaml index 2dc99db9bf..a1ffab814e 100644 --- a/compiler/ztests/remove-passops.yaml +++ b/compiler/ztests/remove-passops.yaml @@ -1,7 +1,7 @@ -script: super compile -O -C 'x>1 | pass | pass | x>2 | pass' +script: super compile -O -C 'from foo | x>1 | pass | pass | x>2 | pass' outputs: - name: stdout data: | - reader filter (x>1 and x>2) + file foo filter (x>1 and x>2) | output main diff --git a/compiler/ztests/tail.yaml b/compiler/ztests/tail.yaml index aefff3b78a..2d129e46ed 100644 --- a/compiler/ztests/tail.yaml +++ b/compiler/ztests/tail.yaml @@ -9,11 +9,11 @@ script: | outputs: - name: stdout data: | - reader + null | tail 1 | output main === - reader + null | ( const x = 1 diff --git a/compiler/ztests/udf-implied-where.yaml b/compiler/ztests/udf-implied-where.yaml index 6febdd3383..079ef66ff4 100644 --- a/compiler/ztests/udf-implied-where.yaml +++ b/compiler/ztests/udf-implied-where.yaml @@ -4,7 +4,7 @@ script: | outputs: - name: stdout data: | - reader + null | ( func h(e): ( has(e) diff --git a/docs/libraries/go.md b/docs/libraries/go.md index 712d45f723..fed92aeb93 100644 --- a/docs/libraries/go.md +++ b/docs/libraries/go.md @@ -134,7 +134,7 @@ func main() { if err != nil { log.Fatalln(err) } - q, err := lake.Query(ctx, nil, "from Demo") + q, err := lake.Query(ctx, "from Demo") if err != nil { log.Fatalln(err) } diff --git a/fuzz/fuzz.go b/fuzz/fuzz.go index 383cca5e11..7e29d90cd8 100644 --- a/fuzz/fuzz.go +++ b/fuzz/fuzz.go @@ -109,7 +109,7 @@ func RunQuery(t testing.TB, zctx *super.Context, readers []zio.Reader, querySour // Infer demand // TODO This is a hack and should be replaced by a cleaner interface in CompileQuery. source := data.NewSource(engine, nil) - dag, err := semantic.AnalyzeAddSource(ctx, ast, source, nil) + dag, err := semantic.Analyze(ctx, ast, source, true) if err != nil { t.Skipf("%v", err) } diff --git a/lake/api/api.go b/lake/api/api.go index 8b1e260b3e..4c8ed8509b 100644 --- a/lake/api/api.go +++ b/lake/api/api.go @@ -11,7 +11,6 @@ import ( "github.com/brimdata/super/api/client" "github.com/brimdata/super/lake" "github.com/brimdata/super/lake/pools" - "github.com/brimdata/super/lakeparse" "github.com/brimdata/super/order" "github.com/brimdata/super/zbuf" "github.com/brimdata/super/zio" @@ -22,7 +21,7 @@ import ( type Interface interface { Root() *lake.Root - Query(ctx context.Context, head *lakeparse.Commitish, src string, srcfiles ...string) (zbuf.Scanner, error) + Query(ctx context.Context, src string, srcfiles ...string) (zbuf.Scanner, error) PoolID(ctx context.Context, poolName string) (ksuid.KSUID, error) CommitObject(ctx context.Context, poolID ksuid.KSUID, branchName string) (ksuid.KSUID, error) CreatePool(context.Context, string, order.SortKeys, int, int64) (ksuid.KSUID, error) @@ -55,7 +54,7 @@ func IsLakeService(u string) bool { func LookupPoolByName(ctx context.Context, api Interface, name string) (*pools.Config, error) { b := newBuffer(pools.Config{}) zed := fmt.Sprintf("from :pools | name == '%s'", name) - q, err := api.Query(ctx, nil, zed) + q, err := api.Query(ctx, zed) if err != nil { return nil, err } @@ -79,7 +78,7 @@ func LookupPoolByName(ctx context.Context, api Interface, name string) (*pools.C func GetPools(ctx context.Context, api Interface) ([]*pools.Config, error) { b := newBuffer(pools.Config{}) - q, err := api.Query(ctx, nil, "from :pools") + q, err := api.Query(ctx, "from :pools") if err != nil { return nil, err } @@ -97,7 +96,7 @@ func GetPools(ctx context.Context, api Interface) ([]*pools.Config, error) { func LookupPoolByID(ctx context.Context, api Interface, id ksuid.KSUID) (*pools.Config, error) { b := newBuffer(pools.Config{}) zed := fmt.Sprintf("from :pools | id == hex('%s')", idToHex(id)) - q, err := api.Query(ctx, nil, zed) + q, err := api.Query(ctx, zed) if err != nil { return nil, err } @@ -122,7 +121,7 @@ func LookupPoolByID(ctx context.Context, api Interface, id ksuid.KSUID) (*pools. func LookupBranchByName(ctx context.Context, api Interface, poolName, branchName string) (*lake.BranchMeta, error) { b := newBuffer(lake.BranchMeta{}) zed := fmt.Sprintf("from :branches | pool.name == '%s' branch.name == '%s'", poolName, branchName) - q, err := api.Query(ctx, nil, zed) + q, err := api.Query(ctx, zed) if err != nil { return nil, err } @@ -147,7 +146,7 @@ func LookupBranchByName(ctx context.Context, api Interface, poolName, branchName func LookupBranchByID(ctx context.Context, api Interface, id ksuid.KSUID) (*lake.BranchMeta, error) { b := newBuffer(lake.BranchMeta{}) zed := fmt.Sprintf("from :branches | branch.id == 'hex(%s)'", idToHex(id)) - q, err := api.Query(ctx, nil, zed) + q, err := api.Query(ctx, zed) if err != nil { return nil, err } diff --git a/lake/api/local.go b/lake/api/local.go index 7cb0091a13..0432c16882 100644 --- a/lake/api/local.go +++ b/lake/api/local.go @@ -105,12 +105,12 @@ func (l *local) Compact(ctx context.Context, poolID ksuid.KSUID, branchName stri return exec.Compact(ctx, l.root, pool, branchName, objects, writeVectors, commit.Author, commit.Body, commit.Meta) } -func (l *local) Query(ctx context.Context, head *lakeparse.Commitish, src string, srcfiles ...string) (zbuf.Scanner, error) { +func (l *local) Query(ctx context.Context, src string, srcfiles ...string) (zbuf.Scanner, error) { ast, err := parser.ParseQuery(src, srcfiles...) if err != nil { return nil, err } - q, err := runtime.CompileLakeQuery(ctx, super.NewContext(), l.compiler, ast, head) + q, err := runtime.CompileLakeQuery(ctx, super.NewContext(), l.compiler, ast) if err != nil { return nil, err } diff --git a/lake/api/remote.go b/lake/api/remote.go index d2e5a61856..9bca3a1895 100644 --- a/lake/api/remote.go +++ b/lake/api/remote.go @@ -119,8 +119,8 @@ func (r *remote) Revert(ctx context.Context, poolID ksuid.KSUID, branchName stri return res.Commit, err } -func (r *remote) Query(ctx context.Context, head *lakeparse.Commitish, src string, srcfiles ...string) (zbuf.Scanner, error) { - res, err := r.conn.Query(ctx, head, src, srcfiles...) +func (r *remote) Query(ctx context.Context, src string, srcfiles ...string) (zbuf.Scanner, error) { + res, err := r.conn.Query(ctx, src, srcfiles...) if err != nil { return nil, err } diff --git a/lake/ztests/checkout-dash-p.yaml b/lake/ztests/checkout-dash-p.yaml index 4acddd4844..013f4b9fc3 100644 --- a/lake/ztests/checkout-dash-p.yaml +++ b/lake/ztests/checkout-dash-p.yaml @@ -7,8 +7,7 @@ script: | super db branch -q child super db use -q @child super db load -q b.jsup - super db use -q POOL - super db query -z "*" + super db query -z "from POOL" inputs: - name: a.jsup diff --git a/lake/ztests/debug.yaml b/lake/ztests/debug.yaml index 628d5c5594..cc0463e40f 100644 --- a/lake/ztests/debug.yaml +++ b/lake/ztests/debug.yaml @@ -3,7 +3,7 @@ script: | super db init -q super db create -use -q test echo '{x: "foo"}' | super db load -q - - super db query -z 'debug f"debug: {x}"' + super db query -z 'from test | debug f"debug: {x}"' outputs: - name: stdout diff --git a/lake/ztests/delete-where-missing.yaml b/lake/ztests/delete-where-missing.yaml index b0d2802af3..d4b89e3d1b 100644 --- a/lake/ztests/delete-where-missing.yaml +++ b/lake/ztests/delete-where-missing.yaml @@ -4,9 +4,9 @@ script: | super db create -use -q test super db load -q in.jsup ! super db delete -q -where 'uid==C3UeSqaSOFRReHD68' - super db query -z 'count:=count()' + super db query -z 'from test | count:=count()' super db delete -q -where 'uid=="C3UeSqaSOFRReHD68"' - super db query -z 'count:=count()' + super db query -z 'from test | count:=count()' inputs: - name: in.jsup diff --git a/lake/ztests/delete-where.yaml b/lake/ztests/delete-where.yaml index b7ad6fc86a..4cb29157d1 100644 --- a/lake/ztests/delete-where.yaml +++ b/lake/ztests/delete-where.yaml @@ -6,9 +6,9 @@ script: | super db create -use -q -S 1KB -orderby ts:$order test seq 1000 | super -c '{ts:this-1,s:f"val{this-1}"}' - | super db load -q - super db delete -q -where 'ts > 400 and ts <= 500' - super db query -z 'count:=count()' + super db query -z 'from test | count:=count()' super db delete -q -where 's == "val1" or s == "val999"' - super db query -z 'count:=count()' + super db query -z 'from test | count:=count()' super db drop -f -q test done diff --git a/lake/ztests/delete.yaml b/lake/ztests/delete.yaml index 722ea78a33..f0161fe04d 100644 --- a/lake/ztests/delete.yaml +++ b/lake/ztests/delete.yaml @@ -5,10 +5,10 @@ script: | super db load -q 1.jsup id=$(super db query -f text "from test@main:objects | cut id:=ksuid(id) | tail 1") super db load -q 2.jsup - super db query -z "*" + super db query -z "from test" echo === | tee /dev/stderr super db delete -q $id - super db query -z "*" + super db query -z "from test" echo === | tee /dev/stderr ! super db delete -q 27aaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/lake/ztests/query-error.yaml b/lake/ztests/query-error.yaml index 9c7e14ed3d..ba6856b4c7 100644 --- a/lake/ztests/query-error.yaml +++ b/lake/ztests/query-error.yaml @@ -9,8 +9,8 @@ script: | outputs: - name: stderr data: | - query must include a 'from' operator - cannot resolve unknown HEAD at line 1, column 6: + query text is missing + HEAD: pool not found at line 1, column 6: from HEAD ~~~~ unknown lake metadata type "unknownmeta" in from operator at line 1, column 6: diff --git a/lake/ztests/revert-revert.yaml b/lake/ztests/revert-revert.yaml index c7f3f865ee..4a4663e50f 100644 --- a/lake/ztests/revert-revert.yaml +++ b/lake/ztests/revert-revert.yaml @@ -7,10 +7,10 @@ script: | super db query -z "from test | sort this" r=$(super db revert $a | awk '{print $5}') echo === - super db query -z "sort this" + super db query -z "from test | sort this" super db revert -q $r echo === - super db query -z "sort this" + super db query -z "from test | sort this" inputs: - name: a.jsup diff --git a/lake/ztests/revert.yaml b/lake/ztests/revert.yaml index bdf20e7f4a..0eda7088ae 100644 --- a/lake/ztests/revert.yaml +++ b/lake/ztests/revert.yaml @@ -4,13 +4,13 @@ script: | super db create -use -q test a=$(super db load a.jsup | head -1 | awk '{print $1}') b=$(super db load b.jsup | head -1 | awk '{print $1}') - super db query -z "sort this" + super db query -z "from test | sort this" super db revert -q $a echo === - super db query -z "sort this" + super db query -z "from test | sort this" super db revert -q $b echo === - super db query -z "sort this" + super db query -z "from test | sort this" inputs: - name: a.jsup diff --git a/lake/ztests/seek-index-boundaries.yaml b/lake/ztests/seek-index-boundaries.yaml index 92ba896fb5..ac21413213 100644 --- a/lake/ztests/seek-index-boundaries.yaml +++ b/lake/ztests/seek-index-boundaries.yaml @@ -2,14 +2,13 @@ script: | export SUPER_DB_LAKE=test super db init -q for o in asc desc; do - echo // $o | tee /dev/stderr super db create -use -q -seekstride 1B -orderby ts:$o $o seq 20 | super -c 'yield {ts:this}' - | super db load -q - - source query.sh 'ts == 5' - source query.sh 'ts < 2' - source query.sh 'ts <= 2' - source query.sh 'ts > 19' - source query.sh 'ts >= 19' + source query.sh "from $o | ts == 5" + source query.sh "from $o | ts < 2" + source query.sh "from $o | ts <= 2" + source query.sh "from $o | ts > 19" + source query.sh "from $o | ts >= 19" done inputs: @@ -20,53 +19,49 @@ inputs: outputs: - name: stdout data: | - // asc - // ts == 5 + // from asc | ts == 5 {ts:5} - // ts < 2 + // from asc | ts < 2 {ts:1} - // ts <= 2 + // from asc | ts <= 2 {ts:1} {ts:2} - // ts > 19 + // from asc | ts > 19 {ts:20} - // ts >= 19 + // from asc | ts >= 19 {ts:19} {ts:20} - // desc - // ts == 5 + // from desc | ts == 5 {ts:5} - // ts < 2 + // from desc | ts < 2 {ts:1} - // ts <= 2 + // from desc | ts <= 2 {ts:2} {ts:1} - // ts > 19 + // from desc | ts > 19 {ts:20} - // ts >= 19 + // from desc | ts >= 19 {ts:20} {ts:19} - name: stderr data: | - // asc - // ts == 5 + // from asc | ts == 5 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts < 2 + // from asc | ts < 2 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts <= 2 + // from asc | ts <= 2 {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} - // ts > 19 + // from asc | ts > 19 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts >= 19 + // from asc | ts >= 19 {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} - // desc - // ts == 5 + // from desc | ts == 5 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts < 2 + // from desc | ts < 2 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts <= 2 + // from desc | ts <= 2 {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} - // ts > 19 + // from desc | ts > 19 {bytes_read:2,bytes_matched:2,records_read:1,records_matched:1} - // ts >= 19 + // from desc | ts >= 19 {bytes_read:4,bytes_matched:4,records_read:2,records_matched:2} diff --git a/lake/ztests/seek-index-null.yaml b/lake/ztests/seek-index-null.yaml index 9b8fafaa57..74190af99a 100644 --- a/lake/ztests/seek-index-null.yaml +++ b/lake/ztests/seek-index-null.yaml @@ -5,7 +5,7 @@ script: | echo // $o | tee /dev/stderr super db create -q -seekstride 2KB -orderby ts:$o $o super db load -q -use $o babble.jsup null.jsup - super db query -z -s -use $o "ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" + super db query -z -s "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" done inputs: diff --git a/lake/ztests/seek-index-ts.yaml b/lake/ztests/seek-index-ts.yaml index 4ed5706bf9..f4b884de33 100644 --- a/lake/ztests/seek-index-ts.yaml +++ b/lake/ztests/seek-index-ts.yaml @@ -2,12 +2,11 @@ script: | export SUPER_DB_LAKE=test super db init -q for o in asc desc; do - echo // $o | tee /dev/stderr super db create -use -seekstride 2KB -orderby ts:$o -q $o super db load -q babble.jsup - source query.sh "ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" - source query.sh "ts == 2020-04-21T23:59:26.06326664Z" - source query.sh "ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar'" + source query.sh "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" + source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z" + source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar'" done @@ -22,37 +21,33 @@ inputs: outputs: - name: stdout data: | - // asc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - // ts == 2020-04-21T23:59:26.06326664Z + // from asc | ts == 2020-04-21T23:59:26.06326664Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // desc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z + // from desc | ts == 2020-04-21T23:59:26.06326664Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - name: stderr data: | - // asc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // ts == 2020-04-21T23:59:26.06326664Z + // from asc | ts == 2020-04-21T23:59:26.06326664Z {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} - // desc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // ts == 2020-04-21T23:59:26.06326664Z + // from desc | ts == 2020-04-21T23:59:26.06326664Z {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} diff --git a/lake/ztests/vacuum.yaml b/lake/ztests/vacuum.yaml index 0a1dfd3fd4..7d49ddff1f 100644 --- a/lake/ztests/vacuum.yaml +++ b/lake/ztests/vacuum.yaml @@ -8,7 +8,7 @@ script: | super db vacuum -dryrun super db vacuum -f super db revert -q $r - ! super db query -z '*' + ! super db query -z 'from test' outputs: - name: stdout diff --git a/runtime/compiler.go b/runtime/compiler.go index 2d58046b01..bb5353663b 100644 --- a/runtime/compiler.go +++ b/runtime/compiler.go @@ -14,7 +14,7 @@ import ( type Compiler interface { NewQuery(*Context, *parser.AST, []zio.Reader) (Query, error) - NewLakeQuery(*Context, *parser.AST, int, *lakeparse.Commitish) (Query, error) + NewLakeQuery(*Context, *parser.AST, int) (Query, error) NewLakeDeleteQuery(*Context, *parser.AST, *lakeparse.Commitish) (DeleteQuery, error) } @@ -44,9 +44,9 @@ func CompileQuery(ctx context.Context, zctx *super.Context, c Compiler, ast *par return q, nil } -func CompileLakeQuery(ctx context.Context, zctx *super.Context, c Compiler, ast *parser.AST, head *lakeparse.Commitish) (Query, error) { +func CompileLakeQuery(ctx context.Context, zctx *super.Context, c Compiler, ast *parser.AST) (Query, error) { rctx := NewContext(ctx, zctx) - q, err := c.NewLakeQuery(rctx, ast, 0, head) + q, err := c.NewLakeQuery(rctx, ast, 0) if err != nil { rctx.Cancel() return nil, err diff --git a/runtime/sam/expr/filter_test.go b/runtime/sam/expr/filter_test.go index 0e597634ca..ccdacc5103 100644 --- a/runtime/sam/expr/filter_test.go +++ b/runtime/sam/expr/filter_test.go @@ -7,6 +7,7 @@ import ( "github.com/brimdata/super" "github.com/brimdata/super/compiler" + "github.com/brimdata/super/compiler/data" "github.com/brimdata/super/compiler/parser" "github.com/brimdata/super/runtime" "github.com/brimdata/super/runtime/sam/expr" @@ -54,7 +55,7 @@ func runCasesHelper(t *testing.T, record string, cases []testcase, expectBufferF t.Helper() ast, err := parser.ParseQuery(c.filter) require.NoError(t, err, "filter: %q", c.filter) - job, err := compiler.NewJob(runtime.DefaultContext(), ast, nil, nil) + job, err := compiler.NewJob(runtime.DefaultContext(), ast, &data.Source{}, true) require.NoError(t, err, "filter: %q", c.filter) err = job.Optimize() require.NoError(t, err, "filter: %q", c.filter) diff --git a/runtime/sam/op/meta/ztests/nulls-max.yaml b/runtime/sam/op/meta/ztests/nulls-max.yaml index 50387e11c8..9a3806c29f 100644 --- a/runtime/sam/op/meta/ztests/nulls-max.yaml +++ b/runtime/sam/op/meta/ztests/nulls-max.yaml @@ -11,7 +11,7 @@ script: | echo '{ts:1}' | super db load -q - super db query -z "from $o:objects | drop id, size" echo "// ===" - super db query -z 'head 1' + super db query -z "from $o | head 1" done outputs: diff --git a/service/auth_test.go b/service/auth_test.go index 0001ced203..55f57a4333 100644 --- a/service/auth_test.go +++ b/service/auth_test.go @@ -37,7 +37,7 @@ func TestAuthIdentity(t *testing.T) { core, conn := newCoreWithConfig(t, service.Config{ Auth: authConfig, }) - _, err := conn.Query(context.Background(), nil, "from [pools]") + _, err := conn.Query(context.Background(), "from [pools]") require.Error(t, err) require.Equal(t, 1.0, promCounterValue(core.Registry(), "request_errors_unauthorized_total")) @@ -59,7 +59,7 @@ func TestAuthIdentity(t *testing.T) { UserID: "test_user_id", }, res) - _, err = conn.Query(context.Background(), nil, "from :pools") + _, err = conn.Query(context.Background(), "from :pools") require.NoError(t, err) } diff --git a/service/client_test.go b/service/client_test.go index 61333ee991..372d4aabb4 100644 --- a/service/client_test.go +++ b/service/client_test.go @@ -48,7 +48,7 @@ func (c *testClient) TestBranchGet(id ksuid.KSUID) (config lake.BranchMeta) { } func (c *testClient) TestPoolList() []pools.Config { - r, err := c.Query(context.Background(), nil, "from :pools") + r, err := c.Query(context.Background(), "from :pools") require.NoError(c, err) defer r.Body.Close() var confs []pools.Config @@ -80,7 +80,7 @@ func (c *testClient) TestBranchPost(poolID ksuid.KSUID, payload api.BranchPostRe } func (c *testClient) TestQuery(query string) string { - r, err := c.Connection.Query(context.Background(), nil, query) + r, err := c.Connection.Query(context.Background(), query) require.NoError(c, err) defer r.Body.Close() zr := zngio.NewReader(super.NewContext(), r.Body) diff --git a/service/handlers.go b/service/handlers.go index 08da636bb6..a27ede26d1 100644 --- a/service/handlers.go +++ b/service/handlers.go @@ -62,7 +62,7 @@ func handleQuery(c *Core, w *ResponseWriter, r *Request) { w.Error(srverr.ErrInvalid(err)) return } - flowgraph, err := runtime.CompileLakeQuery(r.Context(), super.NewContext(), c.compiler, ast, &req.Head) + flowgraph, err := runtime.CompileLakeQuery(r.Context(), super.NewContext(), c.compiler, ast) if err != nil { w.Error(srverr.ErrInvalid(err)) return @@ -182,7 +182,7 @@ func handleQueryDescribe(c *Core, w *ResponseWriter, r *Request) { return } src := data.NewSource(storage.NewRemoteEngine(), c.root) - info, err := describe.Analyze(r.Context(), req.Query, src, &req.Head) + info, err := describe.Analyze(r.Context(), req.Query, src) if err != nil { w.Error(srverr.ErrInvalid(err)) return diff --git a/service/ztests/curl-delete-where.yaml b/service/ztests/curl-delete-where.yaml index 3380e8cf8c..5cbd08e7e7 100644 --- a/service/ztests/curl-delete-where.yaml +++ b/service/ztests/curl-delete-where.yaml @@ -7,10 +7,10 @@ script: | curl -s -d '{where:"x <= 4"}' $SUPER_DB_LAKE/pool/test/branch/main/delete | sed -E 's/0x[0-9a-f]{40}/xxx/' echo === - super db query -z '*' + super db query -z 'from test' echo === curl -w 'code %{response_code}\n' -d '{where:"x <= 4"}' $SUPER_DB_LAKE/pool/test/branch/main/delete - super db query -z '*' + super db query -z 'from test' inputs: - name: service.sh diff --git a/service/ztests/curl-query-error.yaml b/service/ztests/curl-query-error.yaml index 08ccf79f31..29c61d6a0c 100644 --- a/service/ztests/curl-query-error.yaml +++ b/service/ztests/curl-query-error.yaml @@ -13,11 +13,11 @@ inputs: outputs: - name: stdout data: | - {"type":"Error","kind":"invalid operation","error":"no pool name given"} + {"type":"Error","kind":"invalid operation","error":"query text is missing"} code 400 - {"type":"Error","kind":"invalid operation","error":"no pool name given"} + {"type":"Error","kind":"invalid operation","error":"query text is missing"} code 400 - {"type":"Error","kind":"invalid operation","error":"cannot resolve unknown HEAD at line 1, column 6:\nfrom HEAD\n ~~~~","compilation_errors":[{"Msg":"cannot resolve unknown HEAD","Pos":5,"End":8}]} + {"type":"Error","kind":"invalid operation","error":"HEAD: pool not found at line 1, column 6:\nfrom HEAD\n ~~~~","compilation_errors":[{"Msg":"HEAD: pool not found","Pos":5,"End":8}]} code 400 {"type":"Error","kind":"invalid operation","error":"unknown lake metadata type \"unknownmeta\" in from operator at line 1, column 6:\nfrom :unknownmeta\n ~~~~~~~~~~~~","compilation_errors":[{"Msg":"unknown lake metadata type \"unknownmeta\" in from operator","Pos":5,"End":16}]} code 400 diff --git a/service/ztests/debug.yaml b/service/ztests/debug.yaml index 938747c206..3019f05c0b 100644 --- a/service/ztests/debug.yaml +++ b/service/ztests/debug.yaml @@ -2,7 +2,7 @@ script: | source service.sh super db create -use -q test echo '{x: "foo"}' | super db load -q - - super db query -z 'debug f"debug: {x}"' + super db query -z 'from test | debug f"debug: {x}"' inputs: - name: service.sh diff --git a/service/ztests/delete-where.yaml b/service/ztests/delete-where.yaml index 87f7bbb212..0facc1ca83 100644 --- a/service/ztests/delete-where.yaml +++ b/service/ztests/delete-where.yaml @@ -5,11 +5,11 @@ script: | super db create -use -q -S 1KB -orderby ts:$order test seq 1000 | super -c '{ts:this-1,s:f"val{this-1}"}' - | super db load -q - super db delete -q -where 'ts > 400 and ts <= 500' - super db query -z 'count()' + super db query -z 'from test | count()' super db delete -q -where 's == "val1" or s == "val999"' - super db query -z 'count()' + super db query -z 'from test | count()' ! super db delete -q -where 's == "val1" or s == "val999"' - super db query -z 'count()' + super db query -z 'from test | count()' super db drop -f -q test done diff --git a/service/ztests/delete.yaml b/service/ztests/delete.yaml index f254c6144d..c8a79c1e60 100644 --- a/service/ztests/delete.yaml +++ b/service/ztests/delete.yaml @@ -4,10 +4,10 @@ script: | super db load -q 1.jsup id=$(super db query -f text "from test@main:objects | cut id:=ksuid(id) | tail 1") super db load -q 2.jsup - super db query -z "*" + super db query -z "from test" echo === | tee /dev/stderr super db delete -q $id - super db query -z "*" + super db query -z "from test" echo === | tee /dev/stderr ! super db delete -q 27aaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/service/ztests/query-describe.yaml b/service/ztests/query-describe.yaml index 48afda3425..6a64c9369a 100644 --- a/service/ztests/query-describe.yaml +++ b/service/ztests/query-describe.yaml @@ -20,10 +20,10 @@ inputs: ) | put foo := "bar" - name: agg.spq data: | - count() by key1:=v1, key2 + from test1 | count() by key1:=v1, key2 - name: agg-no-keys.spq data: | - sum(this) + from test1 | sum(this) - name: two-channels.spq data: | fork ( @@ -32,7 +32,7 @@ inputs: ) - name: agg-sort.spq data: | - sum(this) by foo | sort x + from test1 | sum(this) by foo | sort x - name: scope.spq data: | type port = uint16 @@ -50,14 +50,12 @@ outputs: { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": false + "id": "XXX" }, { "kind": "Pool", "name": "test2", - "id": "XXX", - "inferred": false + "id": "XXX" } ], "channels": [ @@ -80,8 +78,7 @@ outputs: "sources": { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": true + "id": "XXX" }, "channels": [ { @@ -103,8 +100,7 @@ outputs: "sources": { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": true + "id": "XXX" }, "channels": [ { @@ -120,14 +116,12 @@ outputs: { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": false + "id": "XXX" }, { "kind": "Pool", "name": "test2", - "id": "XXX", - "inferred": false + "id": "XXX" } ], "channels": [ @@ -159,8 +153,7 @@ outputs: "sources": { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": true + "id": "XXX" }, "channels": [ { @@ -186,8 +179,7 @@ outputs: "sources": { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": false + "id": "XXX" }, "channels": [ { @@ -214,8 +206,7 @@ outputs: "sources": { "kind": "Pool", "name": "test1", - "id": "XXX", - "inferred": false + "id": "XXX" }, "channels": [ { diff --git a/service/ztests/query-error.yaml b/service/ztests/query-error.yaml index 567e5e61e6..eae67f0e90 100644 --- a/service/ztests/query-error.yaml +++ b/service/ztests/query-error.yaml @@ -12,8 +12,8 @@ inputs: outputs: - name: stderr data: | - status code 400: no pool name given - cannot resolve unknown HEAD at line 1, column 6: + status code 400: query text is missing + HEAD: pool not found at line 1, column 6: from HEAD ~~~~ unknown lake metadata type "unknownmeta" in from operator at line 1, column 6: diff --git a/service/ztests/seek-index-null.yaml b/service/ztests/seek-index-null.yaml index f4230a4a72..1d7cf04d02 100644 --- a/service/ztests/seek-index-null.yaml +++ b/service/ztests/seek-index-null.yaml @@ -4,7 +4,7 @@ script: | echo // $o | tee /dev/stderr super db create -q -seekstride=2KB -orderby ts:$o $o super db load -q -use $o babble.jsup null.jsup - super db query -z -s -use $o "ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" + super db query -z -s "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" done inputs: diff --git a/service/ztests/seek-index.yaml b/service/ztests/seek-index.yaml index 557a1bbdd5..322d16354c 100644 --- a/service/ztests/seek-index.yaml +++ b/service/ztests/seek-index.yaml @@ -4,9 +4,9 @@ script: | echo // $o | tee /dev/stderr super db create -use -seekstride 2KB -orderby ts:$o -q $o super db load -q babble.jsup - source query.sh "ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" - source query.sh "ts == 2020-04-21T23:59:26.06326664Z" - source query.sh "ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar'" + source query.sh "from $o | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z" + source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z" + source query.sh "from $o | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar'" done inputs: @@ -22,36 +22,36 @@ outputs: - name: stdout data: | // asc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} - // ts == 2020-04-21T23:59:26.06326664Z + // from asc | ts == 2020-04-21T23:59:26.06326664Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} // desc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {ts:2020-04-21T23:59:38.0687693Z,s:"topcoating-rhexis",v:415} {ts:2020-04-21T23:59:29.06985813Z,s:"areek-ashless",v:266} {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z + // from desc | ts == 2020-04-21T23:59:26.06326664Z {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {ts:2020-04-21T23:59:26.06326664Z,s:"potbellied-Dedanim",v:230} - name: stderr data: | // asc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from asc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // ts == 2020-04-21T23:59:26.06326664Z + // from asc | ts == 2020-04-21T23:59:26.06326664Z {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from asc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} // desc - // ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z + // from desc | ts >= 2020-04-21T23:59:26.063Z and ts <= 2020-04-21T23:59:38.069Z {bytes_read:16403,bytes_matched:87,records_read:500,records_matched:3} - // ts == 2020-04-21T23:59:26.06326664Z + // from desc | ts == 2020-04-21T23:59:26.06326664Z {bytes_read:8141,bytes_matched:31,records_read:250,records_matched:1} - // ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' + // from desc | ts == 2020-04-21T23:59:26.06326664Z or foo == 'bar' {bytes_read:32889,bytes_matched:31,records_read:1000,records_matched:1} diff --git a/service/ztests/vacuum.yaml b/service/ztests/vacuum.yaml index 6bfd4c13c5..0cd1837a62 100644 --- a/service/ztests/vacuum.yaml +++ b/service/ztests/vacuum.yaml @@ -7,7 +7,7 @@ script: | super db vacuum -dryrun super db vacuum -f super db revert -q $r - ! super db query -z '*' + ! super db query -z 'from test' inputs: - name: service.sh diff --git a/zfmt/ast.go b/zfmt/ast.go index 5230fee868..42224ef6b2 100644 --- a/zfmt/ast.go +++ b/zfmt/ast.go @@ -605,7 +605,7 @@ func (c *canon) op(p ast.Op) { c.expr(p.Expr, "") } default: - c.open("unknown proc: %T", p) + c.open("unknown operator: %T", p) c.close() } } diff --git a/zfmt/dag.go b/zfmt/dag.go index 5fbe6c4fef..a306207c02 100644 --- a/zfmt/dag.go +++ b/zfmt/dag.go @@ -505,6 +505,9 @@ func (c *canonDAG) op(p dag.Op) { c.expr(p.Filter, "") c.write(")") } + case *dag.NullScan: + c.next() + c.write("null") case *dag.FileScan: c.next() c.write("file %s", p.Path) @@ -548,7 +551,7 @@ func (c *canonDAG) op(p dag.Op) { c.write("output %s", p.Name) default: c.next() - c.open("unknown proc: %T", p) + c.open("unknown operator: %T", p) c.close() } } diff --git a/zfmt/ztests/debug.yaml b/zfmt/ztests/debug.yaml index cb866860f2..a53db1698d 100644 --- a/zfmt/ztests/debug.yaml +++ b/zfmt/ztests/debug.yaml @@ -7,7 +7,7 @@ outputs: data: | debug f"debug: {this}" | head 1 - reader + null | mirror ( => yield "debug: "+cast(this, ) diff --git a/zfmt/ztests/join.yaml b/zfmt/ztests/join.yaml index 71f44d5474..771b74ccd2 100644 --- a/zfmt/ztests/join.yaml +++ b/zfmt/ztests/join.yaml @@ -10,7 +10,7 @@ outputs: from "test.jsup" ) on x=x p:=a === - reader + null | fork ( => pass diff --git a/zfmt/ztests/output.yaml b/zfmt/ztests/output.yaml index 58e77b924d..d97c29dc9b 100644 --- a/zfmt/ztests/output.yaml +++ b/zfmt/ztests/output.yaml @@ -6,7 +6,7 @@ script: | outputs: - name: stdout data: | - reader + null | fork ( => output foo @@ -15,7 +15,7 @@ outputs: | output main ) // === - reader + null | switch x ( case "foo" => output foo diff --git a/zfmt/ztests/over.yaml b/zfmt/ztests/over.yaml index 3814ceb5e2..00b9db16e3 100644 --- a/zfmt/ztests/over.yaml +++ b/zfmt/ztests/over.yaml @@ -42,7 +42,7 @@ outputs: search g ) === - reader + null | yield ( over a | where search("b") diff --git a/zfmt/ztests/precedence-dag.yaml b/zfmt/ztests/precedence-dag.yaml index a6cd7d1a88..807d7e6e0d 100644 --- a/zfmt/ztests/precedence-dag.yaml +++ b/zfmt/ztests/precedence-dag.yaml @@ -16,12 +16,12 @@ inputs: outputs: - name: stdout data: | - reader + null | yield (10+2)/8 | output main - reader + null | where (y==2 or x==4) and z==5 | output main - reader + null | where !(ts<=2) | output main diff --git a/zfmt/ztests/switch.yaml b/zfmt/ztests/switch.yaml index dfb95cb525..00f2510b29 100644 --- a/zfmt/ztests/switch.yaml +++ b/zfmt/ztests/switch.yaml @@ -17,7 +17,7 @@ outputs: tail ) === - reader + null | switch ( case search("a") => head 1 @@ -34,7 +34,7 @@ outputs: tail ) === - reader + null | switch ( case search("a") => head 1 diff --git a/zfmt/ztests/type-value.yaml b/zfmt/ztests/type-value.yaml index 5b9f54e4e9..86921bd3c7 100644 --- a/zfmt/ztests/type-value.yaml +++ b/zfmt/ztests/type-value.yaml @@ -6,6 +6,6 @@ outputs: - name: stdout data: | search is() and bar - reader + null | where is(<(uint16,ip)>) and search(80) | output main diff --git a/zfmt/ztests/yield-shortcut.yaml b/zfmt/ztests/yield-shortcut.yaml index a1e9447279..d6c9e21a85 100644 --- a/zfmt/ztests/yield-shortcut.yaml +++ b/zfmt/ztests/yield-shortcut.yaml @@ -20,18 +20,18 @@ outputs: yield |{"foo":1,"bar":2}| yield <(int64,string)>(1) === - reader + null | yield {x:1,...y} | output main - reader + null | yield [1,2,3] | output main - reader + null | yield |["foo","bar"]| | output main - reader + null | yield |{"foo":1,"bar":2}| | output main - reader + null | yield cast(1, <(int64,string)>) | output main From 129759268aded7624079381b0888c9222e826eb9 Mon Sep 17 00:00:00 2001 From: Steven McCanne Date: Fri, 1 Nov 2024 15:18:05 -0700 Subject: [PATCH 2/2] address PR feedback --- compiler/parser/api.go | 4 ++-- compiler/semantic/analyzer.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/parser/api.go b/compiler/parser/api.go index 6dff677816..5a47cd2ccc 100644 --- a/compiler/parser/api.go +++ b/compiler/parser/api.go @@ -26,7 +26,7 @@ func (a *AST) Files() *srcfiles.List { func (a *AST) ConvertToDeleteWhere(pool, branch string) error { if len(a.seq) == 0 { - return errors.New("delete where command requires an expression") + return errors.New("internal error: AST seq cannot be empty") } a.seq.Prepend(&ast.Delete{ Kind: "Delete", @@ -44,7 +44,7 @@ func ParseQuery(query string, filenames ...string) (*AST, error) { return nil, err } if files.Text == "" { - return &AST{seq: []ast.Op{}, files: files}, nil + return &AST{files: files}, nil } p, err := Parse("", []byte(files.Text), Recover(false)) if err != nil { diff --git a/compiler/semantic/analyzer.go b/compiler/semantic/analyzer.go index 47c2975810..7726822ce0 100644 --- a/compiler/semantic/analyzer.go +++ b/compiler/semantic/analyzer.go @@ -28,7 +28,7 @@ func Analyze(ctx context.Context, ast *parser.AST, source *data.Source, extInput } else if extInput { seq.Prepend(&dag.DefaultScan{Kind: "DefaultScan"}) } else { - // This is a local query and there's not external input + // This is a local query and there's no external input // (i.e., no command-line file args) seq.Prepend(&dag.NullScan{Kind: "NullScan"}) }