Catch up with main

.github/workflows/markdown-lint.yaml

@@ -15,10 +15,10 @@ jobs:
     - uses: actions/checkout@v4
     - name: Lint
       run: make markdown-lint
-    - uses: gaurav-nelson/github-action-markdown-link-check@v1
+    - uses: umbrelladocs/action-linkspector@v1
       with:
-        config-file: .github/workflows/markdown-link-check-config.json
-        use-quiet-mode: yes
+        fail_on_error: true
+        filter_mode: nofilter
     - name: Inform Slack users of link check failures
       uses: tiloio/[email protected]
       if: failure() && github.ref_name == 'main'

.linkspector.yml

@@ -0,0 +1,5 @@
+dirs:
+  - .
+useGitIgnore: true
+excludedDirs:
+  - node_modules

.markdownlint.yaml

@@ -11,4 +11,4 @@ whitespace: true
 MD010:
   code_blocks: false # Disallow hard tabs except in code blocks.
 MD033:
-  allowed_elements: ["p"]
+  allowed_elements: ["p","br"]

CHANGELOG.md

@@ -146,7 +146,7 @@
 * Fix an issue where certain ZNG files could not be read and caused a `control` error (#4579)
 * Fix an issue where `zed serve` would exit if it tried to write to a closed socket (#4587)
 * Improve JSON output for Zed [maps](docs/formats/zed.md#24-map) (#4589)
-* Add the [`zed vacuum`](docs/commands/super-db.md#215-vacuum) command (#4577, #4598, #4600)
+* Add the [`zed vacuum`](docs/commands/super-db.md#vacuum) command (#4577, #4598, #4600)
 
 ## v1.7.0
 * Add [`regexp_replace()`](docs/language/functions/regexp_replace.md) function for replacing regular expression matches in a string (#4435, #4449)
@@ -203,7 +203,7 @@
 * Improve handling of errors during [shaping](docs/language/shaping.md) (#4067, #4069)
 * Allow use of a pool name regexp/glob pattern with the [`from` operator](docs/language/operators/from.md) (#4072, #4075)
 * Add [`levenshtein()` function](docs/language/functions/levenshtein.md) for fuzzy string matching (#4104)
-* Allow use of any filter with [`zed delete -where`](docs/commands/super-db.md#24-delete) (#4100, #4124, #4126, #4125, #4127)
+* Allow use of any filter with [`zed delete -where`](docs/commands/super-db.md#delete) (#4100, #4124, #4126, #4125, #4127)
 * Add [`regexp()`](docs/language/functions/regexp.md) function for regular expression searches and capture groups (#4145, #4158)
 * Add [`coalesce()`](docs/language/functions/coalesce.md) function for locating non-null/non-error values (#4172)
 * Add `line` format for sourcing newline-delimited input as strings (#4175)
@@ -235,7 +235,7 @@
 * Allow conversion of time values to other numeric types (#3816)
 * Remove scaling from duration and time conversions (#3809)
 * Add [`over` expressions](docs/language/lateral-subqueries.md#lateral-expressions) (#3797)
-* Add `-where` flag to [`zed delete`](docs/commands/super-db.md#24-delete) (#3791)
+* Add `-where` flag to [`zed delete`](docs/commands/super-db.md#delete) (#3791)
 * Allow base62 object IDs in lake API request bodies (#3783)
 * Remove `let` operator and [`over` operator](docs/language/operators/over.md)'s `as` clause (#3785)
 
@@ -677,7 +677,7 @@ questions.
 * zqd: Add a Python `zqd` API client for use with tools like [JupyterLab](https://jupyterlab.readthedocs.io/en/stable/) (#1564)
 
 ## v0.22.0
-* zq: Change the implementation of the `union` type to conform with the [ZNG spec](docs/formats/bsup.md#3114-union-typedef) (#1245)
+* zq: Change the implementation of the `union` type to conform with the [ZNG spec](docs/formats/bsup.md#215-union-typedef) (#1245)
 * zq: Make options/flags and version reporting consistent across CLI tools (#1249, #1254, #1256, #1296, #1323, #1334, #1328)
 * zqd: Fix an issue that was preventing flows in nanosecond pcaps from opening in Brim (#1243, #1241)
 * zq: Fix an issue where the TZNG reader did not recognize a bad record type as a syntax error (#1260)
@@ -695,7 +695,7 @@ questions.
 * zql: Fix broken links in documentation (#1321, #1339)
 * zst: Introduce the [ZST format](docs/formats/csup.md) for columnar data based on ZNG (#1268, #1338)
 * pcap: Fix an issue where certain pcapng files could fail import with a `bad option length` error (#1341)
-* zql: [Document the `**` operator](docs/language/search-expressions.md) for type-specific searches that look within nested records (#1337)
+* zql: Document the `**` operator for type-specific searches that look within nested records (#1337)
 * zar: Change the archive data file layout to prepare for handing chunk files with overlapping ranges and improved S3 support (#1330)
 * zar: Support archive data files with overlapping time spans (#1348)
 * zqd: Add a page containing guidance for users that directly access the root `zqd` endpoint in a browser (#1350)
@@ -745,7 +745,7 @@ questions.
 * zqd: Fix an issue with excess characters in Space names after upgrade (#1112)
 
 ## v0.19.0
-* zq: ZNG output is now LZ4-compressed by default (#1050, #1064, #1063, [ZNG spec](docs/formats/bsup.md#313-compressed-value-message-block))
+* zq: ZNG output is now LZ4-compressed by default (#1050, #1064, #1063, [ZNG spec](docs/formats/bsup.md#2-the-super-binary-format))
 * zar: Adjust import size threshold to account for compression (#1082)
 * zqd: Support starting `zqd` with datapath set to an S3 path (#1072)
 * zq: Fix an issue with panics during pcap import (#1090)

compiler/ast/ast.go

@@ -133,6 +133,13 @@ type IndexExpr struct {
 	Loc   `json:"loc"`
 }
 
+type IsNullExpr struct {
+	Kind string `json:"kind" unpack:""`
+	Expr Expr   `json:"expr"`
+	Not  bool   `json:"not"`
+	Loc  `json:"loc"`
+}
+
 type SliceExpr struct {
 	Kind string `json:"kind" unpack:""`
 	Expr Expr   `json:"expr"`
@@ -315,6 +322,7 @@ func (*CaseExpr) ExprAST()    {}
 func (*Cast) ExprAST()        {}
 func (*ID) ExprAST()          {}
 func (*IndexExpr) ExprAST()   {}
+func (*IsNullExpr) ExprAST()  {}
 func (*SliceExpr) ExprAST()   {}
 
 func (*Assignment) ExprAST() {}

compiler/ast/unpack.go

@@ -45,6 +45,7 @@ var unpacker = unpack.New(
 	ID{},
 	ImpliedValue{},
 	IndexExpr{},
+	IsNullExpr{},
 	Join{},
 	Load{},
 	Merge{},

compiler/dag/expr.go

@@ -27,11 +27,6 @@ type (
 		Kind  string       `json:"kind" unpack:""`
 		Elems []VectorElem `json:"elems"`
 	}
-	Assignment struct {
-		Kind string `json:"kind" unpack:""`
-		LHS  Expr   `json:"lhs"`
-		RHS  Expr   `json:"rhs"`
-	}
 	// A BadExpr node is a placeholder for an expression containing semantic
 	// errors.
 	BadExpr struct {
@@ -70,6 +65,10 @@ type (
 		Expr  Expr   `json:"expr"`
 		Index Expr   `json:"index"`
 	}
+	IsNullExpr struct {
+		Kind string `json:"kind" unpack:""`
+		Expr Expr   `json:"expr"`
+	}
 	Literal struct {
 		Kind  string `json:"kind" unpack:""`
 		Value string `json:"value"`
@@ -148,6 +147,7 @@ func (*Conditional) ExprDAG()  {}
 func (*Dot) ExprDAG()          {}
 func (*Func) ExprDAG()         {}
 func (*IndexExpr) ExprDAG()    {}
+func (*IsNullExpr) ExprDAG()   {}
 func (*Literal) ExprDAG()      {}
 func (*MapCall) ExprDAG()      {}
 func (*MapExpr) ExprDAG()      {}

compiler/dag/op.go

@@ -294,6 +294,11 @@ func (*Deleter) OpNode() {}
 // Various Op fields
 
 type (
+	Assignment struct {
+		Kind string `json:"kind" unpack:""`
+		LHS  Expr   `json:"lhs"`
+		RHS  Expr   `json:"rhs"`
+	}
 	Case struct {
 		Expr Expr `json:"expr"`
 		Path Seq  `json:"seq"`

compiler/dag/unpack.go

@@ -33,6 +33,7 @@ var unpacker = unpack.New(
 	Head{},
 	HTTPScan{},
 	IndexExpr{},
+	IsNullExpr{},
 	Join{},
 	LakeMetaScan{},
 	Lister{},

compiler/kernel/expr.go

@@ -75,6 +75,8 @@ func (b *Builder) compileExpr(e dag.Expr) (expr.Evaluator, error) {
 		return b.compileCall(*e)
 	case *dag.IndexExpr:
 		return b.compileIndexExpr(e)
+	case *dag.IsNullExpr:
+		return b.compileIsNullExpr(e)
 	case *dag.SliceExpr:
 		return b.compileSliceExpr(e)
 	case *dag.RegexpMatch:
@@ -405,6 +407,14 @@ func (b *Builder) compileIndexExpr(e *dag.IndexExpr) (expr.Evaluator, error) {
 	return expr.NewIndexExpr(b.zctx(), container, index), nil
 }
 
+func (b *Builder) compileIsNullExpr(e *dag.IsNullExpr) (expr.Evaluator, error) {
+	eval, err := b.compileExpr(e.Expr)
+	if err != nil {
+		return nil, err
+	}
+	return expr.NewIsNullExpr(eval), nil
+}
+
 func (b *Builder) compileRegexpMatch(match *dag.RegexpMatch) (expr.Evaluator, error) {
 	e, err := b.compileExpr(match.Expr)
 	if err != nil {

compiler/kernel/filter.go

@@ -36,9 +36,8 @@ func (f *DeleteFilter) AsEvaluator() (expr.Evaluator, error) {
 		return nil, nil
 	}
 	// For a DeleteFilter Evaluator the pushdown gets wrapped in a unary !
-	// expression so we get all values that don't match. We also add a missing
-	// call so if the expression results in an error("missing") the value is
-	// kept.
+	// expression so we get all values that don't match. We also add an error
+	// and null check because we want to keep these values around.
 	return f.builder.compileExpr(&dag.BinaryExpr{
 		Kind: "BinaryExpr",
 		Op:   "or",
@@ -47,10 +46,11 @@ func (f *DeleteFilter) AsEvaluator() (expr.Evaluator, error) {
 			Op:      "!",
 			Operand: f.pushdown,
 		},
-		RHS: &dag.Call{
-			Kind: "Call",
-			Name: "missing",
-			Args: []dag.Expr{f.pushdown},
+		RHS: &dag.BinaryExpr{
+			Kind: "BinaryExpr",
+			Op:   "or",
+			LHS:  &dag.IsNullExpr{Kind: "IsNullExpr", Expr: f.pushdown},
+			RHS:  &dag.Call{Kind: "Call", Name: "is_error", Args: []dag.Expr{f.pushdown}},
 		},
 	})
 }

compiler/kernel/vexpr.go

@@ -35,12 +35,14 @@ func (b *Builder) compileVamExpr(e dag.Expr) (vamexpr.Evaluator, error) {
 		return b.compileVamDotExpr(e)
 	case *dag.IndexExpr:
 		return b.compileVamIndexExpr(e)
+	case *dag.IsNullExpr:
+		return b.compileVamIsNullExpr(e)
 	case *dag.UnaryExpr:
 		return b.compileVamUnary(*e)
 	case *dag.BinaryExpr:
 		return b.compileVamBinary(e)
-	//case *dag.Conditional:
-	//	return b.compileVamConditional(*e)
+	case *dag.Conditional:
+		return b.compileVamConditional(*e)
 	case *dag.Call:
 		return b.compileVamCall(e)
 	//case *dag.RegexpMatch:
@@ -109,15 +111,30 @@ func (b *Builder) compileVamBinary(e *dag.BinaryExpr) (vamexpr.Evaluator, error)
 	}
 }
 
+func (b *Builder) compileVamConditional(node dag.Conditional) (vamexpr.Evaluator, error) {
+	predicate, err := b.compileVamExpr(node.Cond)
+	if err != nil {
+		return nil, err
+	}
+	thenExpr, err := b.compileVamExpr(node.Then)
+	if err != nil {
+		return nil, err
+	}
+	elseExpr, err := b.compileVamExpr(node.Else)
+	if err != nil {
+		return nil, err
+	}
+	return vamexpr.NewConditional(b.zctx(), predicate, thenExpr, elseExpr), nil
+}
+
 func (b *Builder) compileVamUnary(unary dag.UnaryExpr) (vamexpr.Evaluator, error) {
 	e, err := b.compileVamExpr(unary.Operand)
 	if err != nil {
 		return nil, err
 	}
 	switch unary.Op {
-	//XXX TBD
-	//case "-":
-	//	return vamexpr.NewUnaryMinus(b.zctx(), e), nil
+	case "-":
+		return vamexpr.NewUnaryMinus(b.zctx(), e), nil
 	case "!":
 		return vamexpr.NewLogicalNot(b.zctx(), e), nil
 	default:
@@ -145,6 +162,14 @@ func (b *Builder) compileVamIndexExpr(idx *dag.IndexExpr) (vamexpr.Evaluator, er
 	return vamexpr.NewIndexExpr(b.zctx(), e, index), nil
 }
 
+func (b *Builder) compileVamIsNullExpr(idx *dag.IsNullExpr) (vamexpr.Evaluator, error) {
+	e, err := b.compileVamExpr(idx.Expr)
+	if err != nil {
+		return nil, err
+	}
+	return vamexpr.NewIsNull(e), nil
+}
+
 func (b *Builder) compileVamExprs(in []dag.Expr) ([]vamexpr.Evaluator, error) {
 	var exprs []vamexpr.Evaluator
 	for _, e := range in {