vam: Add SQL semantics for nulls in comparisons

Also add support for IS NULL expressions in vector runtime

compiler/kernel/vexpr.go

@@ -35,6 +35,8 @@ func (b *Builder) compileVamExpr(e dag.Expr) (vamexpr.Evaluator, error) {
 		return b.compileVamDotExpr(e)
 	case *dag.IndexExpr:
 		return b.compileVamIndexExpr(e)
+	case *dag.IsNullExpr:
+		return b.compileVamIsNullExpr(e)
 	case *dag.UnaryExpr:
 		return b.compileVamUnary(*e)
 	case *dag.BinaryExpr:
@@ -145,6 +147,14 @@ func (b *Builder) compileVamIndexExpr(idx *dag.IndexExpr) (vamexpr.Evaluator, er
 	return vamexpr.NewIndexExpr(b.zctx(), e, index), nil
 }
 
+func (b *Builder) compileVamIsNullExpr(idx *dag.IsNullExpr) (vamexpr.Evaluator, error) {
+	e, err := b.compileVamExpr(idx.Expr)
+	if err != nil {
+		return nil, err
+	}
+	return vamexpr.NewIsNull(e), nil
+}
+
 func (b *Builder) compileVamExprs(in []dag.Expr) ([]vamexpr.Evaluator, error) {
 	var exprs []vamexpr.Evaluator
 	for _, e := range in {

runtime/vam/expr/coerce.go

@@ -25,9 +25,11 @@ func coerceVals(zctx *super.Context, a, b vector.Any) (vector.Any, vector.Any, v
 		return a, b, nil
 	}
 	if aid == super.IDNull {
+		a = vector.NewConst(super.NewValue(b.Type(), nil), b.Len(), nil)
 		return a, b, nil //XXX
 	}
 	if bid == super.IDNull {
+		b = vector.NewConst(super.NewValue(a.Type(), nil), a.Len(), nil)
 		return a, b, nil //XXX
 	}
 	if !super.IsNumber(aid) || !super.IsNumber(bid) {

runtime/vam/expr/compare.go

@@ -26,13 +26,19 @@ func (c *Compare) Eval(val vector.Any) vector.Any {
 func (c *Compare) eval(vecs ...vector.Any) vector.Any {
 	lhs := vector.Under(vecs[0])
 	rhs := vector.Under(vecs[1])
+	if _, ok := lhs.(*vector.Error); ok {
+		return vecs[0]
+	}
+	if _, ok := rhs.(*vector.Error); ok {
+		return vecs[1]
+	}
+	nulls := vector.Or(vector.NullsOf(lhs), vector.NullsOf(rhs))
 	lhs, rhs, errVal := coerceVals(c.zctx, lhs, rhs)
 	if errVal != nil {
-		return errVal
+		// if incompatible types return false
+		return vector.NewConst(super.False, vecs[0].Len(), nulls)
 	}
 	//XXX need to handle overflow (see sam)
-	//XXX nulls... for primitives we just do the compare but we need
-	// to or the nulls together
 	kind := vector.KindOf(lhs)
 	if kind != vector.KindOf(rhs) {
 		panic("vector kind mismatch after coerce")
@@ -47,7 +53,32 @@ func (c *Compare) eval(vecs ...vector.Any) vector.Any {
 	}
 	f, ok := compareFuncs[vector.FuncCode(c.opCode, kind, lform, rform)]
 	if !ok {
-		return vector.NewStringError(c.zctx, coerce.ErrIncompatibleTypes.Error(), lhs.Len())
+		return vector.NewConst(super.False, lhs.Len(), nulls)
+	}
+	out := f(lhs, rhs)
+	vector.SetNulls(out, nulls)
+	return out
+}
+
+type isNull struct {
+	expr Evaluator
+}
+
+func NewIsNull(e Evaluator) Evaluator {
+	return &isNull{e}
+}
+
+func (i *isNull) Eval(this vector.Any) vector.Any {
+	return vector.Apply(false, i.eval, i.expr.Eval(this))
+}
+
+func (i *isNull) eval(vecs ...vector.Any) vector.Any {
+	vec := vector.Under(vecs[0])
+	if _, ok := vec.(*vector.Error); ok {
+		return vec
+	}
+	if c, ok := vec.(*vector.Const); ok && c.Value().IsNull() {
+		return vector.NewConst(super.True, vec.Len(), nil)
 	}
-	return f(lhs, rhs)
+	return vector.NullsOf(vec)
 }

runtime/ztests/expr/compare-incompatible.yaml

@@ -0,0 +1,12 @@
+zed: yield a == b
+
+vector: true
+
+input: |
+  {a:"1",b:1(uint8)}
+  {a:127.0.0.1,b:<string>}
+
+output: |
+  false
+  false
+

runtime/sam/expr/ztests/compare-null-int64.yaml → runtime/ztests/expr/compare-null-int64.yaml

@@ -1,5 +1,7 @@
 zed: put eq := (a == b), ne := (a != b), lt := (a < b), lte := (a <= b), gt := (a > b), gte := (a >= b)
 
+vector: true
+
 input: |
   {a:1,b:null(int64)}
   {a:1,b:null}

runtime/sam/expr/ztests/compare-null-string.yaml → runtime/ztests/expr/compare-null-string.yaml

@@ -1,9 +1,15 @@
 zed: put eq := (a == b), ne := (a != b)
 
+vector: true
+
 input: |
   {a:"s",b:null(string)}
   {a:"s",b:null}
+  {a:"s"}
+  {b:null}
 
 output: |
   {a:"s",b:null(string),eq:null(bool),ne:null(bool)}
   {a:"s",b:null,eq:null(bool),ne:null(bool)}
+  {a:"s",eq:error("missing"),ne:error("missing")}
+  {b:null,eq:error("missing"),ne:error("missing")}

runtime/sam/expr/ztests/filter-null-with-unset-field.yaml → runtime/ztests/expr/filter-null-with-unset-field.yaml

@@ -1,5 +1,7 @@
 zed: s IS NULL
 
+vector: true
+
 input: |
   {s:"A=B"}
   {s:"A=*"}

runtime/ztests/expr/is-null.yaml

@@ -0,0 +1,17 @@
+# not for vam needs does not properly handle Const bools and errors. Re-enable 
+# the IS NOT NULL test once this works.
+zed: yield this IS NULL #, this IS NOT NULL
+
+vector: true
+
+input: |
+  "foo"
+  null(string)
+  error("missing")
+  error("foo")
+
+output: |
+  false
+  true
+  error("missing")
+  error("foo")

vector/bool.go

@@ -137,6 +137,13 @@ func NullsOf(v Any) *Bool {
 	case *Bytes:
 		return v.Nulls
 	case *Const:
+		if v.Value().IsNull() {
+			out := NewBoolEmpty(v.Len(), nil)
+			for i := range out.Bits {
+				out.Bits[i] = ^uint64(0)
+			}
+			return out
+		}
 		return v.Nulls
 	case *Dict:
 		return v.Nulls
@@ -172,7 +179,7 @@ func NullsOf(v Any) *Bool {
 	panic(v)
 }
 
-func setNulls(v Any, nulls *Bool) {
+func SetNulls(v Any, nulls *Bool) {
 	switch v := v.(type) {
 	case *Array:
 		v.Nulls = nulls
@@ -195,7 +202,7 @@ func setNulls(v Any, nulls *Bool) {
 	case *Map:
 		v.Nulls = nulls
 	case *Named:
-		setNulls(v.Any, nulls)
+		SetNulls(v.Any, nulls)
 	case *Net:
 		v.Nulls = nulls
 	case *Record:

vector/builder.go

@@ -132,7 +132,7 @@ func (n *nullsBuilder) Build() Any {
 	if !n.nulls.IsEmpty() {
 		bits := make([]uint64, (n.n+63)/64)
 		n.nulls.WriteDenseTo(bits)
-		setNulls(vec, NewBool(bits, n.n, nil))
+		SetNulls(vec, NewBool(bits, n.n, nil))
 	}
 	return vec
 }