evalengine: serialize to SQL (#14337)

Signed-off-by: Vicent Marti <[email protected]>

go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go

@@ -31,6 +31,9 @@ import (
 )
 
 func start(t *testing.T) (utils.MySQLCompare, func()) {
+	// ensure that the vschema and the tables have been created before running any tests
+	_ = utils.WaitForAuthoritative(t, keyspaceName, "t1", clusterInstance.VtgateProcess.ReadVSchema)
+
 	mcmp, err := utils.NewMySQLCompare(t, vtParams, mysqlParams)
 	require.NoError(t, err)
 

go/tools/sizegen/sizegen.go

@@ -115,7 +115,8 @@ func isPod(tt types.Type) bool {
 func (sizegen *sizegen) getKnownType(named *types.Named) *typeState {
 	ts := sizegen.known[named]
 	if ts == nil {
-		local := strings.HasPrefix(named.Obj().Pkg().Path(), sizegen.mod.Path)
+		pkg := named.Obj().Pkg()
+		local := pkg != nil && strings.HasPrefix(pkg.Path(), sizegen.mod.Path)
 		ts = &typeState{
 			local: local,
 			pod:   isPod(named.Underlying()),

go/vt/sqlparser/ast.go

@@ -3139,7 +3139,7 @@ type (
 	}
 )
 
-// iExpr ensures that only expressions nodes can be assigned to a Expr
+// IsExpr ensures that only expressions nodes can be assigned to a Expr
 func (*AndExpr) IsExpr()                            {}
 func (*OrExpr) IsExpr()                             {}
 func (*XorExpr) IsExpr()                            {}

go/vt/sqlparser/tracked_buffer.go

@@ -86,6 +86,10 @@ func (buf *TrackedBuffer) SetUpperCase(enable bool) {
 	}
 }
 
+func (buf *TrackedBuffer) WriteLiteral(lit string) {
+	_, _ = buf.literal(lit)
+}
+
 // SetEscapeAllIdentifiers sets whether ALL identifiers in the serialized SQL query should be quoted
 // and escaped. By default, identifiers are only escaped if they match the name of a SQL keyword or they
 // contain characters that must be escaped.

go/vt/vtgate/engine/delete.go

@@ -20,9 +20,8 @@ import (
 	"context"
 	"fmt"
 
-	"vitess.io/vitess/go/vt/vtgate/evalengine"
-
 	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
+	"vitess.io/vitess/go/vt/sqlparser"
 
 	"vitess.io/vitess/go/sqltypes"
 	"vitess.io/vitess/go/vt/srvtopo"
@@ -155,7 +154,7 @@ func addFieldsIfNotEmpty(dml *DML, other map[string]any) {
 	if len(dml.Values) > 0 {
 		s := []string{}
 		for _, value := range dml.Values {
-			s = append(s, evalengine.FormatExpr(value))
+			s = append(s, sqlparser.String(value))
 		}
 		other["Values"] = s
 	}

go/vt/vtgate/engine/insert.go

@@ -971,7 +971,7 @@ func (ins *Insert) description() PrimitiveDescription {
 			for _, exprs := range ints {
 				var this []string
 				for _, expr := range exprs {
-					this = append(this, evalengine.FormatExpr(expr))
+					this = append(this, sqlparser.String(expr))
 				}
 				res = append(res, strings.Join(this, ", "))
 			}
@@ -985,7 +985,7 @@ func (ins *Insert) description() PrimitiveDescription {
 		if ins.Generate.Values == nil {
 			other["AutoIncrement"] = fmt.Sprintf("%s:Offset(%d)", ins.Generate.Query, ins.Generate.Offset)
 		} else {
-			other["AutoIncrement"] = fmt.Sprintf("%s:Values::%s", ins.Generate.Query, evalengine.FormatExpr(ins.Generate.Values))
+			other["AutoIncrement"] = fmt.Sprintf("%s:Values::%s", ins.Generate.Query, sqlparser.String(ins.Generate.Values))
 		}
 	}
 

go/vt/vtgate/engine/limit.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"strconv"
 
+	"vitess.io/vitess/go/vt/sqlparser"
 	"vitess.io/vitess/go/vt/vtgate/evalengine"
 
 	"vitess.io/vitess/go/sqltypes"
@@ -204,10 +205,10 @@ func (l *Limit) description() PrimitiveDescription {
 	other := map[string]any{}
 
 	if l.Count != nil {
-		other["Count"] = evalengine.FormatExpr(l.Count)
+		other["Count"] = sqlparser.String(l.Count)
 	}
 	if l.Offset != nil {
-		other["Offset"] = evalengine.FormatExpr(l.Offset)
+		other["Offset"] = sqlparser.String(l.Offset)
 	}
 
 	return PrimitiveDescription{

go/vt/vtgate/engine/plan_description_test.go

@@ -19,6 +19,7 @@ package engine
 import (
 	"testing"
 
+	"vitess.io/vitess/go/vt/sqlparser"
 	"vitess.io/vitess/go/vt/vtgate/evalengine"
 
 	"vitess.io/vitess/go/test/utils"
@@ -80,8 +81,8 @@ func TestPlanDescriptionWithInputs(t *testing.T) {
 	expected := PrimitiveDescription{
 		OperatorType: "Limit",
 		Other: map[string]any{
-			"Count":  evalengine.FormatExpr(count),
-			"Offset": evalengine.FormatExpr(offset),
+			"Count":  sqlparser.String(count),
+			"Offset": sqlparser.String(offset),
 		},
 		Inputs: []PrimitiveDescription{routeDescr},
 	}

go/vt/vtgate/engine/projection.go

@@ -21,9 +21,9 @@ import (
 	"sync"
 
 	"vitess.io/vitess/go/mysql"
-	"vitess.io/vitess/go/mysql/collations"
 	"vitess.io/vitess/go/sqltypes"
 	querypb "vitess.io/vitess/go/vt/proto/query"
+	"vitess.io/vitess/go/vt/sqlparser"
 	"vitess.io/vitess/go/vt/vtgate/evalengine"
 )
 
@@ -74,7 +74,7 @@ func (p *Projection) TryExecute(ctx context.Context, vcursor VCursor, bindVars m
 		resultRows = append(resultRows, resultRow)
 	}
 	if wantfields {
-		result.Fields, err = p.evalFields(env, result.Fields, vcursor)
+		result.Fields, err = p.evalFields(env, result.Fields)
 		if err != nil {
 			return nil, err
 		}
@@ -92,7 +92,7 @@ func (p *Projection) TryStreamExecute(ctx context.Context, vcursor VCursor, bind
 		var err error
 		if wantfields {
 			once.Do(func() {
-				fields, err = p.evalFields(env, qr.Fields, vcursor)
+				fields, err = p.evalFields(env, qr.Fields)
 				if err != nil {
 					return
 				}
@@ -131,33 +131,32 @@ func (p *Projection) GetFields(ctx context.Context, vcursor VCursor, bindVars ma
 		return nil, err
 	}
 	env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor)
-	qr.Fields, err = p.evalFields(env, qr.Fields, vcursor)
+	qr.Fields, err = p.evalFields(env, qr.Fields)
 	if err != nil {
 		return nil, err
 	}
 	return qr, nil
 }
 
-func (p *Projection) evalFields(env *evalengine.ExpressionEnv, infields []*querypb.Field, vcursor VCursor) ([]*querypb.Field, error) {
+func (p *Projection) evalFields(env *evalengine.ExpressionEnv, infields []*querypb.Field) ([]*querypb.Field, error) {
+	// TODO: once the evalengine becomes smart enough, we should be able to remove the
+	// dependency on these fields altogether
+	env.Fields = infields
+
 	var fields []*querypb.Field
 	for i, col := range p.Cols {
-		q, f, err := env.TypeOf(p.Exprs[i], infields)
+		typ, err := env.TypeOf(p.Exprs[i])
 		if err != nil {
 			return nil, err
 		}
-		var cs collations.ID = collations.CollationBinaryID
-		if sqltypes.IsText(q) {
-			cs = vcursor.ConnCollation()
-		}
-
-		fl := mysql.FlagsForColumn(q, cs)
-		if !sqltypes.IsNull(q) && !f.Nullable() {
+		fl := mysql.FlagsForColumn(typ.Type, typ.Coll)
+		if !sqltypes.IsNull(typ.Type) && !typ.Nullable {
 			fl |= uint32(querypb.MySqlFlag_NOT_NULL_FLAG)
 		}
 		fields = append(fields, &querypb.Field{
 			Name:    col,
-			Type:    q,
-			Charset: uint32(cs),
+			Type:    typ.Type,
+			Charset: uint32(typ.Coll),
 			Flags:   fl,
 		})
 	}
@@ -173,7 +172,7 @@ func (p *Projection) Inputs() ([]Primitive, []map[string]any) {
 func (p *Projection) description() PrimitiveDescription {
 	var exprs []string
 	for idx, e := range p.Exprs {
-		expr := evalengine.FormatExpr(e)
+		expr := sqlparser.String(e)
 		alias := p.Cols[idx]
 		if alias != "" {
 			expr += " as " + alias

go/vt/vtgate/engine/route.go

@@ -29,6 +29,7 @@ import (
 	"vitess.io/vitess/go/mysql/collations"
 	"vitess.io/vitess/go/mysql/sqlerror"
 	"vitess.io/vitess/go/vt/log"
+	"vitess.io/vitess/go/vt/sqlparser"
 	"vitess.io/vitess/go/vt/vtgate/evalengine"
 
 	"vitess.io/vitess/go/sqltypes"
@@ -466,7 +467,7 @@ func (route *Route) description() PrimitiveDescription {
 	if route.Values != nil {
 		formattedValues := make([]string, 0, len(route.Values))
 		for _, value := range route.Values {
-			formattedValues = append(formattedValues, evalengine.FormatExpr(value))
+			formattedValues = append(formattedValues, sqlparser.String(value))
 		}
 		other["Values"] = formattedValues
 	}
@@ -476,15 +477,15 @@ func (route *Route) description() PrimitiveDescription {
 			if idx != 0 {
 				sysTabSchema += ", "
 			}
-			sysTabSchema += evalengine.FormatExpr(tableSchema)
+			sysTabSchema += sqlparser.String(tableSchema)
 		}
 		sysTabSchema += "]"
 		other["SysTableTableSchema"] = sysTabSchema
 	}
 	if len(route.SysTableTableName) != 0 {
 		var sysTableName []string
 		for k, v := range route.SysTableTableName {
-			sysTableName = append(sysTableName, k+":"+evalengine.FormatExpr(v))
+			sysTableName = append(sysTableName, k+":"+sqlparser.String(v))
 		}
 		sort.Strings(sysTableName)
 		other["SysTableTableName"] = "[" + strings.Join(sysTableName, ", ") + "]"

go/vt/vtgate/engine/set.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"strings"
 
+	"vitess.io/vitess/go/vt/sqlparser"
 	"vitess.io/vitess/go/vt/sysvars"
 
 	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"
@@ -178,7 +179,7 @@ func (u *UserDefinedVariable) MarshalJSON() ([]byte, error) {
 	}{
 		Type: "UserDefinedVariable",
 		Name: u.Name,
-		Expr: evalengine.FormatExpr(u.Expr),
+		Expr: sqlparser.String(u.Expr),
 	})
 
 }
@@ -439,7 +440,7 @@ func (svss *SysVarSetAware) MarshalJSON() ([]byte, error) {
 	}{
 		Type: "SysVarAware",
 		Name: svss.Name,
-		Expr: evalengine.FormatExpr(svss.Expr),
+		Expr: sqlparser.String(svss.Expr),
 	})
 }
 

go/vt/vtgate/engine/set_test.go

@@ -107,7 +107,7 @@ func TestSetTable(t *testing.T) {
 		setOps: []SetOp{
 			&UserDefinedVariable{
 				Name: "x",
-				Expr: evalengine.NewColumn(0, evalengine.UnknownType()),
+				Expr: evalengine.NewColumn(0, evalengine.UnknownType(), nil),
 			},
 		},
 		qr: []*sqltypes.Result{sqltypes.MakeTestResult(

go/vt/vtgate/engine/vindex_func.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"vitess.io/vitess/go/vt/sqlparser"
 	"vitess.io/vitess/go/vt/vterrors"
 	"vitess.io/vitess/go/vt/vtgate/evalengine"
 
@@ -245,7 +246,7 @@ func (vf *VindexFunc) description() PrimitiveDescription {
 	other := map[string]any{
 		"Fields":  fields,
 		"Columns": vf.Cols,
-		"Value":   evalengine.FormatExpr(vf.Value),
+		"Value":   sqlparser.String(vf.Value),
 	}
 	if vf.Vindex != nil {
 		other["Vindex"] = vf.Vindex.String()

go/vt/vtgate/engine/vindex_lookup.go

@@ -20,6 +20,7 @@ import (
 	"context"
 
 	vtgatepb "vitess.io/vitess/go/vt/proto/vtgate"
+	"vitess.io/vitess/go/vt/sqlparser"
 
 	"vitess.io/vitess/go/vt/key"
 
@@ -150,7 +151,7 @@ func (vr *VindexLookup) description() PrimitiveDescription {
 	if vr.Values != nil {
 		formattedValues := make([]string, 0, len(vr.Values))
 		for _, value := range vr.Values {
-			formattedValues = append(formattedValues, evalengine.FormatExpr(value))
+			formattedValues = append(formattedValues, sqlparser.String(value))
 		}
 		other["Values"] = formattedValues
 	}