Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[release-17.0] planbuilder bugfix: expose columns through derived tables (#14501) #14503

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -224,3 +224,71 @@ func TestInfrSchemaAndUnionAll(t *testing.T) {
})
}
}
<<<<<<< HEAD
=======

func TestTypeORMQuery(t *testing.T) {
// This test checks that we can run queries similar to the ones that the TypeORM framework uses

require.NoError(t,
utils.WaitForAuthoritative(t, "ks", "t1", clusterInstance.VtgateProcess.ReadVSchema))

mcmp, closer := start(t)
defer closer()

query := `SELECT kcu.TABLE_NAME, kcu.COLUMN_NAME, cols.DATA_TYPE
FROM (SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME
FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
WHERE kcu.TABLE_SCHEMA = 'ks'
AND kcu.TABLE_NAME = 't1'
UNION
SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME
FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
WHERE kcu.TABLE_SCHEMA = 'ks'
AND kcu.TABLE_NAME = 't7_xxhash') kcu
INNER JOIN (SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, DATA_TYPE
FROM INFORMATION_SCHEMA.COLUMNS cols
WHERE cols.TABLE_SCHEMA = 'ks'
AND cols.TABLE_NAME = 't1'
UNION
SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, DATA_TYPE
FROM INFORMATION_SCHEMA.COLUMNS cols
WHERE cols.TABLE_SCHEMA = 'ks'
AND cols.TABLE_NAME = 't7_xxhash') cols
ON kcu.TABLE_SCHEMA = cols.TABLE_SCHEMA AND kcu.TABLE_NAME = cols.TABLE_NAME AND
kcu.COLUMN_NAME = cols.COLUMN_NAME`
utils.AssertMatchesAny(t, mcmp.VtConn, query,
`[[VARBINARY("t1") VARCHAR("id1") BLOB("bigint")] [VARBINARY("t7_xxhash") VARCHAR("uid") BLOB("varchar")]]`,
`[[VARCHAR("t1") VARCHAR("id1") BLOB("bigint")] [VARCHAR("t7_xxhash") VARCHAR("uid") BLOB("varchar")]]`,
)
}

func TestJoinWithSingleShardQueryOnRHS(t *testing.T) {
// This test checks that we can run queries like this, where the RHS is a single shard query
mcmp, closer := start(t)
defer closer()

query := `SELECT
c.column_name as column_name,
c.data_type as data_type,
c.table_name as table_name,
c.table_schema as table_schema
FROM
information_schema.columns c
JOIN (
SELECT
table_name
FROM
information_schema.tables
WHERE
table_schema != 'information_schema'
LIMIT
1
) AS tables ON tables.table_name = c.table_name
ORDER BY
c.table_name`

res := utils.Exec(t, mcmp.VtConn, query)
require.NotEmpty(t, res.Rows)
}
>>>>>>> 817c24e942 (planbuilder bugfix: expose columns through derived tables (#14501))
13 changes: 13 additions & 0 deletions go/vt/vtgate/planbuilder/operators/aggregator.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,22 @@ func (a *Aggregator) isDerived() bool {
return a.TableID != nil
}

<<<<<<< HEAD
func (a *Aggregator) AddColumn(ctx *plancontext.PlanningContext, expr *sqlparser.AliasedExpr, _, addToGroupBy bool) (ops.Operator, int, error) {
if addToGroupBy {
return nil, 0, vterrors.VT13001("did not expect to add group by here")
=======
func (a *Aggregator) FindCol(ctx *plancontext.PlanningContext, in sqlparser.Expr, underRoute bool) int {
if underRoute && a.isDerived() {
// We don't want to use columns on this operator if it's a derived table under a route.
// In this case, we need to add a Projection on top of this operator to make the column available
return -1
}

expr := a.DT.RewriteExpression(ctx, in)
if offset, found := canReuseColumn(ctx, a.Columns, expr, extractExpr); found {
return offset
>>>>>>> 817c24e942 (planbuilder bugfix: expose columns through derived tables (#14501))
}
if offset, found := canReuseColumn(ctx, a.Columns, expr.Expr, extractExpr); found {
return a, offset, nil
Expand Down
109 changes: 109 additions & 0 deletions go/vt/vtgate/planbuilder/operators/horizon.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,115 @@ func (h *Horizon) SetInputs(ops []ops.Operator) {
h.Source = ops[0]
}

<<<<<<< HEAD
=======
func (h *Horizon) AddPredicate(ctx *plancontext.PlanningContext, expr sqlparser.Expr) ops.Operator {
if _, isUNion := h.Source.(*Union); isUNion {
// If we have a derived table on top of a UNION, we can let the UNION do the expression rewriting
h.Source = h.Source.AddPredicate(ctx, expr)
return h
}
tableInfo, err := ctx.SemTable.TableInfoForExpr(expr)
if err != nil {
if errors.Is(err, semantics.ErrNotSingleTable) {
return &Filter{
Source: h,
Predicates: []sqlparser.Expr{expr},
}
}
panic(err)
}

newExpr := semantics.RewriteDerivedTableExpression(expr, tableInfo)
if sqlparser.ContainsAggregation(newExpr) {
return &Filter{Source: h, Predicates: []sqlparser.Expr{expr}}
}
h.Source = h.Source.AddPredicate(ctx, newExpr)
return h
}

func (h *Horizon) AddColumn(ctx *plancontext.PlanningContext, reuse bool, _ bool, expr *sqlparser.AliasedExpr) int {
if !reuse {
panic(errNoNewColumns)
}
col, ok := expr.Expr.(*sqlparser.ColName)
if !ok {
panic(vterrors.VT13001("cannot push non-ColName expression to horizon"))
}
offset := h.FindCol(ctx, col, false)
if offset < 0 {
panic(errNoNewColumns)
}
return offset
}

var errNoNewColumns = vterrors.VT13001("can't add new columns to Horizon")

// canReuseColumn is generic, so it can be used with slices of different types.
// We don't care about the actual type, as long as we know it's a sqlparser.Expr
func canReuseColumn[T any](
ctx *plancontext.PlanningContext,
columns []T,
col sqlparser.Expr,
f func(T) sqlparser.Expr,
) (offset int, found bool) {
for offset, column := range columns {
if ctx.SemTable.EqualsExprWithDeps(col, f(column)) {
return offset, true
}
}

return
}

func (h *Horizon) FindCol(ctx *plancontext.PlanningContext, expr sqlparser.Expr, underRoute bool) int {
if underRoute && h.IsDerived() {
// We don't want to use columns on this operator if it's a derived table under a route.
// In this case, we need to add a Projection on top of this operator to make the column available
return -1
}

for idx, se := range sqlparser.GetFirstSelect(h.Query).SelectExprs {
ae, ok := se.(*sqlparser.AliasedExpr)
if !ok {
panic(vterrors.VT09015())
}
if ctx.SemTable.EqualsExprWithDeps(ae.Expr, expr) {
return idx
}
}

return -1
}

func (h *Horizon) GetColumns(ctx *plancontext.PlanningContext) (exprs []*sqlparser.AliasedExpr) {
for _, expr := range ctx.SemTable.SelectExprs(h.Query) {
ae, ok := expr.(*sqlparser.AliasedExpr)
if !ok {
panic(vterrors.VT09015())
}
exprs = append(exprs, ae)
}

return exprs
}

func (h *Horizon) GetSelectExprs(*plancontext.PlanningContext) sqlparser.SelectExprs {
return sqlparser.GetFirstSelect(h.Query).SelectExprs
}

func (h *Horizon) GetOrdering(ctx *plancontext.PlanningContext) []ops.OrderBy {
if h.QP == nil {
_, err := h.getQP(ctx)
if err != nil {
panic(err)
}
}
return h.QP.OrderExprs
}

// TODO: REMOVE
>>>>>>> 817c24e942 (planbuilder bugfix: expose columns through derived tables (#14501))
func (h *Horizon) selectStatement() sqlparser.SelectStatement {
return h.Select
}
Expand Down
8 changes: 6 additions & 2 deletions go/vt/vtgate/planbuilder/operators/route_planning.go
Original file line number Diff line number Diff line change
Expand Up @@ -412,19 +412,23 @@ func mergeOrJoin(ctx *plancontext.PlanningContext, lhs, rhs ops.Operator, joinPr

if len(joinPredicates) > 0 && requiresSwitchingSides(ctx, rhs) {
if !inner {
return nil, nil, vterrors.VT12001("LEFT JOIN with derived tables")
return nil, nil, vterrors.VT12001("LEFT JOIN with LIMIT on the outer side")
}

if requiresSwitchingSides(ctx, lhs) {
return nil, nil, vterrors.VT12001("JOIN between derived tables")
return nil, nil, vterrors.VT12001("JOIN between derived tables with LIMIT")
}

join := NewApplyJoin(Clone(rhs), Clone(lhs), nil, !inner)
newOp, err := pushJoinPredicates(ctx, joinPredicates, join)
if err != nil {
return nil, nil, err
}
<<<<<<< HEAD
return newOp, rewrite.NewTree("merge routes, but switch sides", newOp), nil
=======
return newOp, rewrite.NewTree("logical join to applyJoin, switching side because LIMIT", newOp), nil
>>>>>>> 817c24e942 (planbuilder bugfix: expose columns through derived tables (#14501))
}

join := NewApplyJoin(Clone(lhs), Clone(rhs), nil, !inner)
Expand Down
Loading