From 5002c3a726c25e5a5b0b15d8a0b396c0792a4309 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Sat, 14 Oct 2023 18:09:49 +0530 Subject: [PATCH 01/43] store child foreign key to update exprs mapping Signed-off-by: Harshit Gangal --- go/vt/vtgate/semantics/analyzer.go | 36 +++-- go/vt/vtgate/semantics/analyzer_test.go | 188 ++++++++++------------- go/vt/vtgate/semantics/semantic_state.go | 1 + 3 files changed, 104 insertions(+), 121 deletions(-) diff --git a/go/vt/vtgate/semantics/analyzer.go b/go/vt/vtgate/semantics/analyzer.go index aaf33e98a8b..16933186f9a 100644 --- a/go/vt/vtgate/semantics/analyzer.go +++ b/go/vt/vtgate/semantics/analyzer.go @@ -107,7 +107,7 @@ func (a *analyzer) newSemTable(statement sqlparser.Statement, coll collations.ID columns[union] = info.exprs } - childFks, parentFks, err := a.getInvolvedForeignKeys(statement) + childFks, parentFks, childFkToUpdExprs, err := a.getInvolvedForeignKeys(statement) if err != nil { return nil, err } @@ -129,6 +129,7 @@ func (a *analyzer) newSemTable(statement sqlparser.Statement, coll collations.ID QuerySignature: a.sig, childForeignKeysInvolved: childFks, parentForeignKeysInvolved: parentFks, + ChildFkToUpdExprs: childFkToUpdExprs, }, nil } @@ -320,14 +321,14 @@ func (a *analyzer) noteQuerySignature(node sqlparser.SQLNode) { } // getInvolvedForeignKeys gets the foreign keys that might require taking care off when executing the given statement. -func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, error) { +func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs, error) { // There are only the DML statements that require any foreign keys handling. switch stmt := statement.(type) { case *sqlparser.Delete: // For DELETE statements, none of the parent foreign keys require handling. // So we collect all the child foreign keys. allChildFks, _, err := a.getAllManagedForeignKeys() - return allChildFks, nil, err + return allChildFks, nil, nil, err case *sqlparser.Insert: // For INSERT statements, we have 3 different cases: // 1. REPLACE statement: REPLACE statements are essentially DELETEs and INSERTs rolled into one. @@ -337,35 +338,35 @@ func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[Ta // 3. INSERT with ON DUPLICATE KEY UPDATE: This might trigger an update on the columns specified in the ON DUPLICATE KEY UPDATE clause. allChildFks, allParentFKs, err := a.getAllManagedForeignKeys() if err != nil { - return nil, nil, err + return nil, nil, nil, err } if stmt.Action == sqlparser.ReplaceAct { - return allChildFks, allParentFKs, nil + return allChildFks, allParentFKs, nil, nil } if len(stmt.OnDup) == 0 { - return nil, allParentFKs, nil + return nil, allParentFKs, nil, nil } // If only a certain set of columns are being updated, then there might be some child foreign keys that don't need any consideration since their columns aren't being updated. // So, we filter these child foreign keys out. We can't filter any parent foreign keys because the statement will INSERT a row too, which requires validating all the parent foreign keys. - updatedChildFks, _ := a.filterForeignKeysUsingUpdateExpressions(allChildFks, nil, sqlparser.UpdateExprs(stmt.OnDup)) - return updatedChildFks, allParentFKs, nil + updatedChildFks, _, childFkToUpdExprs := a.filterForeignKeysUsingUpdateExpressions(allChildFks, nil, sqlparser.UpdateExprs(stmt.OnDup)) + return updatedChildFks, allParentFKs, childFkToUpdExprs, nil case *sqlparser.Update: // For UPDATE queries we get all the parent and child foreign keys, but we can filter some of them out if the columns that they consist off aren't being updated or are set to NULLs. allChildFks, allParentFks, err := a.getAllManagedForeignKeys() if err != nil { - return nil, nil, err + return nil, nil, nil, err } - childFks, parentFks := a.filterForeignKeysUsingUpdateExpressions(allChildFks, allParentFks, stmt.Exprs) - return childFks, parentFks, nil + childFks, parentFks, childFkToUpdExprs := a.filterForeignKeysUsingUpdateExpressions(allChildFks, allParentFks, stmt.Exprs) + return childFks, parentFks, childFkToUpdExprs, nil default: - return nil, nil, nil + return nil, nil, nil, nil } } // filterForeignKeysUsingUpdateExpressions filters the child and parent foreign key constraints that don't require any validations/cascades given the updated expressions. -func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[TableSet][]vindexes.ChildFKInfo, allParentFks map[TableSet][]vindexes.ParentFKInfo, updExprs sqlparser.UpdateExprs) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo) { +func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[TableSet][]vindexes.ChildFKInfo, allParentFks map[TableSet][]vindexes.ParentFKInfo, updExprs sqlparser.UpdateExprs) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs) { if len(allChildFks) == 0 && len(allParentFks) == 0 { - return nil, nil + return nil, nil, nil } pFksRequired := make(map[TableSet][]bool, len(allParentFks)) @@ -380,6 +381,9 @@ func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[Table // updExprToTableSet stores the tables that the updated expressions are from. updExprToTableSet := make(map[*sqlparser.ColName]TableSet) + // childFKToUpdExprs stores child foreign key to update expressions mapping. + childFKToUpdExprs := map[string]sqlparser.UpdateExprs{} + // Go over all the update expressions for _, updateExpr := range updExprs { deps := a.binder.direct.dependencies(updateExpr.Name) @@ -396,6 +400,8 @@ func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[Table for idx, childFk := range childFks { if childFk.ParentColumns.FindColumn(updateExpr.Name.Name) >= 0 { cFksRequired[deps][idx] = true + tbl, _ := a.tables.tableInfoFor(deps) + childFKToUpdExprs[childFk.String(tbl.GetVindexTable())] = append(childFKToUpdExprs[childFk.String(tbl.GetVindexTable())], updateExpr) } } // If we are setting a column to NULL, then we don't need to verify the existance of an @@ -449,7 +455,7 @@ func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[Table cFksNeedsHandling[ts] = cFKNeeded } - return cFksNeedsHandling, pFksNeedsHandling + return cFksNeedsHandling, pFksNeedsHandling, childFKToUpdExprs } // getAllManagedForeignKeys gets all the foreign keys for the query we are analyzing that Vitess is reposible for managing. diff --git a/go/vt/vtgate/semantics/analyzer_test.go b/go/vt/vtgate/semantics/analyzer_test.go index dd2d80d62d4..19f60c3a4ed 100644 --- a/go/vt/vtgate/semantics/analyzer_test.go +++ b/go/vt/vtgate/semantics/analyzer_test.go @@ -1554,6 +1554,59 @@ var tbl = map[string]TableInfo{ Keyspace: &vindexes.Keyspace{Name: "undefined_ks", Sharded: true}, }, }, + "t4": &RealTable{ + Table: &vindexes.Table{ + Keyspace: &vindexes.Keyspace{Name: "ks"}, + ChildForeignKeys: []vindexes.ChildFKInfo{ + ckInfo(nil, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(nil, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), + ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + }, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(nil, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), + pkInfo(nil, []string{"pcolc"}, []string{"colc"}), + pkInfo(nil, []string{"pcolb", "pcola"}, []string{"colb", "cola"}), + pkInfo(nil, []string{"pcolb"}, []string{"colb"}), + pkInfo(nil, []string{"pcola"}, []string{"cola"}), + pkInfo(nil, []string{"pcolb", "pcolx"}, []string{"colb", "colx"}), + }, + }, + }, + "t5": &RealTable{ + Table: &vindexes.Table{ + Keyspace: &vindexes.Keyspace{Name: "ks"}, + ChildForeignKeys: []vindexes.ChildFKInfo{ + ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + ckInfo(nil, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), + ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), + }, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(nil, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), + pkInfo(nil, []string{"pcola"}, []string{"cola"}), + pkInfo(nil, []string{"pcold", "pcolc"}, []string{"cold", "colc"}), + pkInfo(nil, []string{"pcold"}, []string{"cold"}), + pkInfo(nil, []string{"pcold", "pcolx"}, []string{"cold", "colx"}), + }, + }, + }, + "t6": &RealTable{ + Table: &vindexes.Table{ + Keyspace: &vindexes.Keyspace{Name: "ks"}, + ChildForeignKeys: []vindexes.ChildFKInfo{ + ckInfo(nil, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(nil, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + ckInfo(nil, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(nil, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), + ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + }, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(nil, []string{"colb"}, []string{"colb"}), + pkInfo(nil, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + }, + }, + }, } // TestGetAllManagedForeignKeys tests the functionality of getAllManagedForeignKeys. @@ -1569,7 +1622,9 @@ func TestGetAllManagedForeignKeys(t *testing.T) { name: "Collect all foreign key constraints", analyzer: &analyzer{ tables: &tableCollector{ - Tables: []TableInfo{tbl["t0"], tbl["t1"], + Tables: []TableInfo{ + tbl["t0"], + tbl["t1"], &DerivedTable{}, }, si: &FakeSI{ @@ -1639,6 +1694,17 @@ func TestFilterForeignKeysUsingUpdateExpressions(t *testing.T) { cold: SingleTableSet(1), }, }, + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t4"], + tbl["t5"], + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_FK_MANAGED, + }, + }, + }, } updateExprs := sqlparser.UpdateExprs{ &sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}, @@ -1660,17 +1726,8 @@ func TestFilterForeignKeysUsingUpdateExpressions(t *testing.T) { analyzer: a, allParentFks: nil, allChildFks: map[TableSet][]vindexes.ChildFKInfo{ - SingleTableSet(0): { - ckInfo(nil, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(nil, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), - ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), - ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - }, - SingleTableSet(1): { - ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - ckInfo(nil, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), - ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), - }, + SingleTableSet(0): tbl["t4"].(*RealTable).Table.ChildForeignKeys, + SingleTableSet(1): tbl["t5"].(*RealTable).Table.ChildForeignKeys, }, updExprs: updateExprs, childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ @@ -1688,21 +1745,8 @@ func TestFilterForeignKeysUsingUpdateExpressions(t *testing.T) { name: "Parent Foreign Keys Filtering", analyzer: a, allParentFks: map[TableSet][]vindexes.ParentFKInfo{ - SingleTableSet(0): { - pkInfo(nil, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), - pkInfo(nil, []string{"pcolc"}, []string{"colc"}), - pkInfo(nil, []string{"pcolb", "pcola"}, []string{"colb", "cola"}), - pkInfo(nil, []string{"pcolb"}, []string{"colb"}), - pkInfo(nil, []string{"pcola"}, []string{"cola"}), - pkInfo(nil, []string{"pcolb", "pcolx"}, []string{"colb", "colx"}), - }, - SingleTableSet(1): { - pkInfo(nil, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), - pkInfo(nil, []string{"pcola"}, []string{"cola"}), - pkInfo(nil, []string{"pcold", "pcolc"}, []string{"cold", "colc"}), - pkInfo(nil, []string{"pcold"}, []string{"cold"}), - pkInfo(nil, []string{"pcold", "pcolx"}, []string{"cold", "colx"}), - }, + SingleTableSet(0): tbl["t4"].(*RealTable).Table.ParentForeignKeys, + SingleTableSet(1): tbl["t5"].(*RealTable).Table.ParentForeignKeys, }, allChildFks: nil, updExprs: updateExprs, @@ -1720,7 +1764,7 @@ func TestFilterForeignKeysUsingUpdateExpressions(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - childFks, parentFks := tt.analyzer.filterForeignKeysUsingUpdateExpressions(tt.allChildFks, tt.allParentFks, tt.updExprs) + childFks, parentFks, _ := tt.analyzer.filterForeignKeysUsingUpdateExpressions(tt.allChildFks, tt.allParentFks, tt.updExprs) require.EqualValues(t, tt.childFksWanted, childFks) require.EqualValues(t, tt.parentFksWanted, parentFks) }) @@ -1769,22 +1813,10 @@ func TestGetInvolvedForeignKeys(t *testing.T) { name: "Update statement", stmt: &sqlparser.Update{ Exprs: sqlparser.UpdateExprs{ - &sqlparser.UpdateExpr{ - Name: cola, - Expr: sqlparser.NewIntLiteral("1"), - }, - &sqlparser.UpdateExpr{ - Name: colb, - Expr: &sqlparser.NullVal{}, - }, - &sqlparser.UpdateExpr{ - Name: colc, - Expr: sqlparser.NewIntLiteral("1"), - }, - &sqlparser.UpdateExpr{ - Name: cold, - Expr: &sqlparser.NullVal{}, - }, + &sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}, + &sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}, + &sqlparser.UpdateExpr{Name: colc, Expr: sqlparser.NewIntLiteral("1")}, + &sqlparser.UpdateExpr{Name: cold, Expr: &sqlparser.NullVal{}}, }, }, analyzer: &analyzer{ @@ -1798,42 +1830,8 @@ func TestGetInvolvedForeignKeys(t *testing.T) { }, tables: &tableCollector{ Tables: []TableInfo{ - &RealTable{ - Table: &vindexes.Table{ - Keyspace: &vindexes.Keyspace{Name: "ks"}, - ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(nil, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(nil, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), - ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), - ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - }, - ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(nil, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), - pkInfo(nil, []string{"pcolc"}, []string{"colc"}), - pkInfo(nil, []string{"pcolb", "pcola"}, []string{"colb", "cola"}), - pkInfo(nil, []string{"pcolb"}, []string{"colb"}), - pkInfo(nil, []string{"pcola"}, []string{"cola"}), - pkInfo(nil, []string{"pcolb", "pcolx"}, []string{"colb", "colx"}), - }, - }, - }, - &RealTable{ - Table: &vindexes.Table{ - Keyspace: &vindexes.Keyspace{Name: "ks"}, - ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - ckInfo(nil, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), - ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), - }, - ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(nil, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), - pkInfo(nil, []string{"pcola"}, []string{"cola"}), - pkInfo(nil, []string{"pcold", "pcolc"}, []string{"cold", "colc"}), - pkInfo(nil, []string{"pcold"}, []string{"cold"}), - pkInfo(nil, []string{"pcold", "pcolx"}, []string{"cold", "colx"}), - }, - }, - }, + tbl["t4"], + tbl["t5"], }, si: &FakeSI{ KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ @@ -1926,14 +1924,8 @@ func TestGetInvolvedForeignKeys(t *testing.T) { stmt: &sqlparser.Insert{ Action: sqlparser.InsertAct, OnDup: sqlparser.OnDup{ - &sqlparser.UpdateExpr{ - Name: cola, - Expr: sqlparser.NewIntLiteral("1"), - }, - &sqlparser.UpdateExpr{ - Name: colb, - Expr: &sqlparser.NullVal{}, - }, + &sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}, + &sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}, }, }, analyzer: &analyzer{ @@ -1945,23 +1937,7 @@ func TestGetInvolvedForeignKeys(t *testing.T) { }, tables: &tableCollector{ Tables: []TableInfo{ - &RealTable{ - Table: &vindexes.Table{ - Keyspace: &vindexes.Keyspace{Name: "ks"}, - ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(nil, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(nil, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), - ckInfo(nil, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(nil, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), - ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), - ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - }, - ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(nil, []string{"colb"}, []string{"colb"}), - pkInfo(nil, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), - }, - }, - }, + tbl["t6"], tbl["t1"], }, si: &FakeSI{ @@ -2024,7 +2000,7 @@ func TestGetInvolvedForeignKeys(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - childFks, parentFks, err := tt.analyzer.getInvolvedForeignKeys(tt.stmt) + childFks, parentFks, _, err := tt.analyzer.getInvolvedForeignKeys(tt.stmt) if tt.expectedErr != "" { require.EqualError(t, err, tt.expectedErr) return diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 4e31d7ebd8e..16d77ec1080 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -131,6 +131,7 @@ type ( // The map is keyed by the tableset of the table that each of the foreign key belongs to. childForeignKeysInvolved map[TableSet][]vindexes.ChildFKInfo parentForeignKeysInvolved map[TableSet][]vindexes.ParentFKInfo + ChildFkToUpdExprs map[string]sqlparser.UpdateExprs } columnName struct { From 929b7b7f249026871d186b7a5e150109bbce0e18 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Sat, 14 Oct 2023 18:38:06 +0530 Subject: [PATCH 02/43] refactor: add only unique columns to select expressions Signed-off-by: Harshit Gangal --- .../vtgate/planbuilder/operators/ast_to_op.go | 11 -------- go/vt/vtgate/planbuilder/operators/delete.go | 6 ++-- go/vt/vtgate/planbuilder/operators/update.go | 28 +++++++++++++++++-- .../testdata/foreignkey_cases.json | 12 ++++---- 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/ast_to_op.go b/go/vt/vtgate/planbuilder/operators/ast_to_op.go index 83f5d268094..5a377b877bb 100644 --- a/go/vt/vtgate/planbuilder/operators/ast_to_op.go +++ b/go/vt/vtgate/planbuilder/operators/ast_to_op.go @@ -410,14 +410,3 @@ func createSelectionOp( // There are no foreign keys to check for a select query, so we can pass anything for verifyAllFKs and fkToIgnore. return createOpFromStmt(ctx, selectionStmt, false /* verifyAllFKs */, "" /* fkToIgnore */) } - -func selectParentColumns(fk vindexes.ChildFKInfo, lastOffset int) ([]int, []sqlparser.SelectExpr) { - var cols []int - var exprs []sqlparser.SelectExpr - for _, column := range fk.ParentColumns { - cols = append(cols, lastOffset) - exprs = append(exprs, aeWrap(sqlparser.NewColName(column.String()))) - lastOffset++ - } - return cols, exprs -} diff --git a/go/vt/vtgate/planbuilder/operators/delete.go b/go/vt/vtgate/planbuilder/operators/delete.go index 128c625c428..f14138f5547 100644 --- a/go/vt/vtgate/planbuilder/operators/delete.go +++ b/go/vt/vtgate/planbuilder/operators/delete.go @@ -174,10 +174,10 @@ func createFkCascadeOpForDelete(ctx *plancontext.PlanningContext, parentOp ops.O } // We need to select all the parent columns for the foreign key constraint, to use in the update of the child table. - cols, exprs := selectParentColumns(fk, len(selectExprs)) - selectExprs = append(selectExprs, exprs...) + var offsets []int + offsets, selectExprs = addColumns(ctx, fk.ParentColumns, selectExprs) - fkChild, err := createFkChildForDelete(ctx, fk, cols) + fkChild, err := createFkChildForDelete(ctx, fk, offsets) if err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 252e5dd6195..4fc70aaf6b8 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -264,10 +264,10 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, } // We need to select all the parent columns for the foreign key constraint, to use in the update of the child table. - cols, exprs := selectParentColumns(fk, len(selectExprs)) - selectExprs = append(selectExprs, exprs...) + var offsets []int + offsets, selectExprs = addColumns(ctx, fk.ParentColumns, selectExprs) - fkChild, err := createFkChildForUpdate(ctx, fk, updStmt, cols, updatedTable) + fkChild, err := createFkChildForUpdate(ctx, fk, updStmt, offsets, updatedTable) if err != nil { return nil, err } @@ -286,6 +286,28 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, }, nil } +func addColumns(ctx *plancontext.PlanningContext, columns sqlparser.Columns, exprs []sqlparser.SelectExpr) ([]int, []sqlparser.SelectExpr) { + var offsets []int + selectExprs := exprs + for _, column := range columns { + ae := aeWrap(sqlparser.NewColName(column.String())) + exists := false + for idx, expr := range exprs { + if ctx.SemTable.EqualsExpr(expr.(*sqlparser.AliasedExpr).Expr, ae.Expr) { + offsets = append(offsets, idx) + exists = true + break + } + } + if !exists { + offsets = append(offsets, len(selectExprs)) + selectExprs = append(selectExprs, ae) + + } + } + return offsets, selectExprs +} + // createFkChildForUpdate creates the update query operator for the child table based on the foreign key constraints. func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, cols []int, updatedTable *vindexes.Table) (*FkChild, error) { // Create a ValTuple of child column names diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index 0d3c5e4745a..c689803f258 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -656,8 +656,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select col1, col1 from u_tbl1 where 1 != 1", - "Query": "select col1, col1 from u_tbl1 for update", + "FieldQuery": "select col1 from u_tbl1 where 1 != 1", + "Query": "select col1 from u_tbl1 for update", "Table": "u_tbl1" }, { @@ -715,7 +715,7 @@ "OperatorType": "FkCascade", "BvName": "fkc_vals2", "Cols": [ - 1 + 0 ], "Inputs": [ { @@ -872,8 +872,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select col1, col1 from u_tbl1 where 1 != 1", - "Query": "select col1, col1 from u_tbl1 where id = 1 for update", + "FieldQuery": "select col1 from u_tbl1 where 1 != 1", + "Query": "select col1 from u_tbl1 where id = 1 for update", "Table": "u_tbl1" }, { @@ -931,7 +931,7 @@ "OperatorType": "FkCascade", "BvName": "fkc_vals2", "Cols": [ - 1 + 0 ], "Inputs": [ { From fab5e3f2e499a7ac051356bc0517194a17ec7768 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 18 Oct 2023 15:21:46 +0530 Subject: [PATCH 03/43] test: improve the fuzzer test to generate expressions containing columns and literals for update queries Signed-off-by: Manan Gupta --- .../vtgate/foreignkey/fk_fuzz_test.go | 60 ++++++++----------- .../endtoend/vtgate/foreignkey/utils_test.go | 29 +++++++++ 2 files changed, 55 insertions(+), 34 deletions(-) diff --git a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go index 134b9cfa180..c645aba98f0 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go @@ -131,41 +131,33 @@ func (fz *fuzzer) generateInsertDMLQuery() string { if tableName == "fk_t20" { colValue := rand.Intn(1 + fz.maxValForCol) col2Value := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("insert into %v (id, col, col2) values (%v, %v, %v)", tableName, idValue, convertColValueToString(colValue), convertColValueToString(col2Value)) + return fmt.Sprintf("insert into %v (id, col, col2) values (%v, %v, %v)", tableName, idValue, convertIntValueToString(colValue), convertIntValueToString(col2Value)) } else if isMultiColFkTable(tableName) { colaValue := rand.Intn(1 + fz.maxValForCol) colbValue := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("insert into %v (id, cola, colb) values (%v, %v, %v)", tableName, idValue, convertColValueToString(colaValue), convertColValueToString(colbValue)) + return fmt.Sprintf("insert into %v (id, cola, colb) values (%v, %v, %v)", tableName, idValue, convertIntValueToString(colaValue), convertIntValueToString(colbValue)) } else { colValue := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("insert into %v (id, col) values (%v, %v)", tableName, idValue, convertColValueToString(colValue)) + return fmt.Sprintf("insert into %v (id, col) values (%v, %v)", tableName, idValue, convertIntValueToString(colValue)) } } -// convertColValueToString converts the given value to a string -func convertColValueToString(value int) string { - if value == 0 { - return "NULL" - } - return fmt.Sprintf("%d", value) -} - // generateUpdateDMLQuery generates an UPDATE query from the parameters for the fuzzer. func (fz *fuzzer) generateUpdateDMLQuery() string { tableId := rand.Intn(len(fkTables)) idValue := 1 + rand.Intn(fz.maxValForId) tableName := fkTables[tableId] if tableName == "fk_t20" { - colValue := rand.Intn(1 + fz.maxValForCol) - col2Value := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("update %v set col = %v, col2 = %v where id = %v", tableName, convertColValueToString(colValue), convertColValueToString(col2Value), idValue) + colValue := fz.generateExpression(rand.Intn(4)+1, "col", "col2") + col2Value := fz.generateExpression(rand.Intn(4)+1, "col", "col2") + return fmt.Sprintf("update %v set col = %v, col2 = %v where id = %v", tableName, colValue, col2Value, idValue) } else if isMultiColFkTable(tableName) { - colaValue := rand.Intn(1 + fz.maxValForCol) - colbValue := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("update %v set cola = %v, colb = %v where id = %v", tableName, convertColValueToString(colaValue), convertColValueToString(colbValue), idValue) + colaValue := fz.generateExpression(rand.Intn(4)+1, "cola", "colb") + colbValue := fz.generateExpression(rand.Intn(4)+1, "cola", "colb") + return fmt.Sprintf("update %v set cola = %v, colb = %v where id = %v", tableName, colaValue, colbValue, idValue) } else { colValue := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("update %v set col = %v where id = %v", tableName, convertColValueToString(colValue), idValue) + return fmt.Sprintf("update %v set col = %v where id = %v", tableName, convertIntValueToString(colValue), idValue) } } @@ -338,8 +330,8 @@ func (fz *fuzzer) getPreparedInsertQueries() []string { return []string{ "prepare stmt_insert from 'insert into fk_t20 (id, col, col2) values (?, ?, ?)'", fmt.Sprintf("SET @id = %v", idValue), - fmt.Sprintf("SET @col = %v", convertColValueToString(colValue)), - fmt.Sprintf("SET @col2 = %v", convertColValueToString(col2Value)), + fmt.Sprintf("SET @col = %v", convertIntValueToString(colValue)), + fmt.Sprintf("SET @col2 = %v", convertIntValueToString(col2Value)), "execute stmt_insert using @id, @col, @col2", } } else if isMultiColFkTable(tableName) { @@ -348,8 +340,8 @@ func (fz *fuzzer) getPreparedInsertQueries() []string { return []string{ fmt.Sprintf("prepare stmt_insert from 'insert into %v (id, cola, colb) values (?, ?, ?)'", tableName), fmt.Sprintf("SET @id = %v", idValue), - fmt.Sprintf("SET @cola = %v", convertColValueToString(colaValue)), - fmt.Sprintf("SET @colb = %v", convertColValueToString(colbValue)), + fmt.Sprintf("SET @cola = %v", convertIntValueToString(colaValue)), + fmt.Sprintf("SET @colb = %v", convertIntValueToString(colbValue)), "execute stmt_insert using @id, @cola, @colb", } } else { @@ -357,7 +349,7 @@ func (fz *fuzzer) getPreparedInsertQueries() []string { return []string{ fmt.Sprintf("prepare stmt_insert from 'insert into %v (id, col) values (?, ?)'", tableName), fmt.Sprintf("SET @id = %v", idValue), - fmt.Sprintf("SET @col = %v", convertColValueToString(colValue)), + fmt.Sprintf("SET @col = %v", convertIntValueToString(colValue)), "execute stmt_insert using @id, @col", } } @@ -374,8 +366,8 @@ func (fz *fuzzer) getPreparedUpdateQueries() []string { return []string{ "prepare stmt_update from 'update fk_t20 set col = ?, col2 = ? where id = ?'", fmt.Sprintf("SET @id = %v", idValue), - fmt.Sprintf("SET @col = %v", convertColValueToString(colValue)), - fmt.Sprintf("SET @col2 = %v", convertColValueToString(col2Value)), + fmt.Sprintf("SET @col = %v", convertIntValueToString(colValue)), + fmt.Sprintf("SET @col2 = %v", convertIntValueToString(col2Value)), "execute stmt_update using @col, @col2, @id", } } else if isMultiColFkTable(tableName) { @@ -384,8 +376,8 @@ func (fz *fuzzer) getPreparedUpdateQueries() []string { return []string{ fmt.Sprintf("prepare stmt_update from 'update %v set cola = ?, colb = ? where id = ?'", tableName), fmt.Sprintf("SET @id = %v", idValue), - fmt.Sprintf("SET @cola = %v", convertColValueToString(colaValue)), - fmt.Sprintf("SET @colb = %v", convertColValueToString(colbValue)), + fmt.Sprintf("SET @cola = %v", convertIntValueToString(colaValue)), + fmt.Sprintf("SET @colb = %v", convertIntValueToString(colbValue)), "execute stmt_update using @cola, @colb, @id", } } else { @@ -393,7 +385,7 @@ func (fz *fuzzer) getPreparedUpdateQueries() []string { return []string{ fmt.Sprintf("prepare stmt_update from 'update %v set col = ? where id = ?'", tableName), fmt.Sprintf("SET @id = %v", idValue), - fmt.Sprintf("SET @col = %v", convertColValueToString(colValue)), + fmt.Sprintf("SET @col = %v", convertIntValueToString(colValue)), "execute stmt_update using @col, @id", } } @@ -419,14 +411,14 @@ func (fz *fuzzer) generateParameterizedInsertQuery() (query string, params []any if tableName == "fk_t20" { colValue := rand.Intn(1 + fz.maxValForCol) col2Value := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("insert into %v (id, col, col2) values (?, ?, ?)", tableName), []any{idValue, convertColValueToString(colValue), convertColValueToString(col2Value)} + return fmt.Sprintf("insert into %v (id, col, col2) values (?, ?, ?)", tableName), []any{idValue, convertIntValueToString(colValue), convertIntValueToString(col2Value)} } else if isMultiColFkTable(tableName) { colaValue := rand.Intn(1 + fz.maxValForCol) colbValue := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("insert into %v (id, cola, colb) values (?, ?, ?)", tableName), []any{idValue, convertColValueToString(colaValue), convertColValueToString(colbValue)} + return fmt.Sprintf("insert into %v (id, cola, colb) values (?, ?, ?)", tableName), []any{idValue, convertIntValueToString(colaValue), convertIntValueToString(colbValue)} } else { colValue := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("insert into %v (id, col) values (?, ?)", tableName), []any{idValue, convertColValueToString(colValue)} + return fmt.Sprintf("insert into %v (id, col) values (?, ?)", tableName), []any{idValue, convertIntValueToString(colValue)} } } @@ -438,14 +430,14 @@ func (fz *fuzzer) generateParameterizedUpdateQuery() (query string, params []any if tableName == "fk_t20" { colValue := rand.Intn(1 + fz.maxValForCol) col2Value := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("update %v set col = ?, col2 = ? where id = ?", tableName), []any{convertColValueToString(colValue), convertColValueToString(col2Value), idValue} + return fmt.Sprintf("update %v set col = ?, col2 = ? where id = ?", tableName), []any{convertIntValueToString(colValue), convertIntValueToString(col2Value), idValue} } else if isMultiColFkTable(tableName) { colaValue := rand.Intn(1 + fz.maxValForCol) colbValue := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("update %v set cola = ?, colb = ? where id = ?", tableName), []any{convertColValueToString(colaValue), convertColValueToString(colbValue), idValue} + return fmt.Sprintf("update %v set cola = ?, colb = ? where id = ?", tableName), []any{convertIntValueToString(colaValue), convertIntValueToString(colbValue), idValue} } else { colValue := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("update %v set col = ? where id = ?", tableName), []any{convertColValueToString(colValue), idValue} + return fmt.Sprintf("update %v set col = ? where id = ?", tableName), []any{convertIntValueToString(colValue), idValue} } } diff --git a/go/test/endtoend/vtgate/foreignkey/utils_test.go b/go/test/endtoend/vtgate/foreignkey/utils_test.go index 5e0b4a8a3cc..d81a40eb90b 100644 --- a/go/test/endtoend/vtgate/foreignkey/utils_test.go +++ b/go/test/endtoend/vtgate/foreignkey/utils_test.go @@ -19,6 +19,7 @@ package foreignkey import ( "database/sql" "fmt" + "math/rand" "strings" "testing" @@ -28,6 +29,8 @@ import ( "vitess.io/vitess/go/test/endtoend/utils" ) +var supportedOpps = []string{"*", "+", "-"} + // getTestName prepends whether the test is for a sharded keyspace or not to the test name. func getTestName(testName string, testSharded bool) string { if testSharded { @@ -41,6 +44,32 @@ func isMultiColFkTable(tableName string) bool { return strings.Contains(tableName, "multicol") } +func (fz *fuzzer) generateExpression(length int, cols ...string) string { + expr := fz.getColOrInt(cols...) + if length == 1 { + return expr + } + rhsExpr := fz.generateExpression(length-1, cols...) + op := supportedOpps[rand.Intn(len(supportedOpps))] + return fmt.Sprintf("%v %s (%v)", expr, op, rhsExpr) +} + +// getColOrInt gets a column or an integer/NULL literal with equal probability. +func (fz *fuzzer) getColOrInt(cols ...string) string { + if len(cols) == 0 || rand.Intn(2) == 0 { + return convertIntValueToString(rand.Intn(1 + fz.maxValForCol)) + } + return cols[rand.Intn(len(cols))] +} + +// convertIntValueToString converts the given value to a string +func convertIntValueToString(value int) string { + if value == 0 { + return "NULL" + } + return fmt.Sprintf("%d", value) +} + // waitForSchemaTrackingForFkTables waits for schema tracking to have run and seen the tables used // for foreign key tests. func waitForSchemaTrackingForFkTables(t *testing.T) { From f7734cd9d81cc457a14d267fc5fa0106db7d8d70 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Oct 2023 12:05:16 +0530 Subject: [PATCH 04/43] feat: add basic planning and execution support for update non-literal queries Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/cached_size.go | 17 +- go/vt/vtgate/engine/fk_cascade.go | 111 ++++++++-- .../planbuilder/operator_transformers.go | 9 +- .../vtgate/planbuilder/operators/ast_to_op.go | 1 + .../planbuilder/operators/fk_cascade.go | 18 +- go/vt/vtgate/planbuilder/operators/update.go | 127 +++++++++--- .../testdata/foreignkey_cases.json | 195 +++++++++++++++++- 7 files changed, 421 insertions(+), 57 deletions(-) diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 10d862ea3df..2ccb8a3307a 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -280,7 +280,7 @@ func (cached *FkChild) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(64) + size += int64(128) } // field BVName string size += hack.RuntimeAllocSize(int64(len(cached.BVName))) @@ -288,6 +288,21 @@ func (cached *FkChild) CachedSize(alloc bool) int64 { { size += hack.RuntimeAllocSize(int64(cap(cached.Cols)) * int64(8)) } + // field UpdateExprBvNames []string + { + size += hack.RuntimeAllocSize(int64(cap(cached.UpdateExprBvNames)) * int64(16)) + for _, elem := range cached.UpdateExprBvNames { + size += hack.RuntimeAllocSize(int64(len(elem))) + } + } + // field UpdateExprCols []int + { + size += hack.RuntimeAllocSize(int64(cap(cached.UpdateExprCols)) * int64(8)) + } + // field CompExprCols []int + { + size += hack.RuntimeAllocSize(int64(cap(cached.CompExprCols)) * int64(8)) + } // field Exec vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Exec.(cachedObject); ok { size += cc.CachedSize(true) diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index e7d14d0aa31..c872c7f401e 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -29,9 +29,12 @@ import ( // FkChild contains the Child Primitive to be executed collecting the values from the Selection Primitive using the column indexes. // BVName is used to pass the value as bind variable to the Child Primitive. type FkChild struct { - BVName string - Cols []int // indexes - Exec Primitive + BVName string + Cols []int // indexes + UpdateExprBvNames []string + UpdateExprCols []int + CompExprCols []int + Exec Primitive } // FkCascade is a primitive that implements foreign key cascading using Selection as values required to execute the FkChild Primitives. @@ -82,35 +85,97 @@ func (fkc *FkCascade) TryExecute(ctx context.Context, vcursor VCursor, bindVars } for _, child := range fkc.Children { + if len(child.UpdateExprBvNames) > 0 { + err = fkc.executeNonLiteralUpdateFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child) + } else { + err = fkc.executeLiteralUpdateFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child) + } + if err != nil { + return nil, err + } + } + + // All the children are modified successfully, we can now execute the Parent Primitive. + return vcursor.ExecutePrimitive(ctx, fkc.Parent, bindVars, wantfields) +} + +func (fkc *FkCascade) executeLiteralUpdateFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild) error { + // We create a bindVariable for each Child + // that stores the tuple of columns involved in the fk constraint. + bv := &querypb.BindVariable{ + Type: querypb.Type_TUPLE, + } + for _, row := range selectionRes.Rows { + // Create a tuple from each Row. + tuple := &querypb.Value{ + Type: querypb.Type_TUPLE, + } + for _, colIdx := range child.Cols { + tuple.Values = append(tuple.Values, + sqltypes.ValueToProto(row[colIdx])) + } + bv.Values = append(bv.Values, tuple) + } + // Execute the child primitive, and bail out incase of failure. + // Since this Primitive is always executed in a transaction, the changes should + // be rolled back incase of an error. + bindVars[child.BVName] = bv + _, err := vcursor.ExecutePrimitive(ctx, child.Exec, bindVars, wantfields) + if err != nil { + return err + } + delete(bindVars, child.BVName) + return nil +} + +func (fkc *FkCascade) executeNonLiteralUpdateFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild) error { + for _, row := range selectionRes.Rows { + skipRow := true + for _, colIdx := range child.CompExprCols { + hasChanged, err := row[colIdx].ToBool() + if err != nil { + return err + } + if hasChanged { + skipRow = false + break + } + } + if skipRow { + continue + } // We create a bindVariable for each Child // that stores the tuple of columns involved in the fk constraint. bv := &querypb.BindVariable{ Type: querypb.Type_TUPLE, } - for _, row := range selectionRes.Rows { - // Create a tuple from each Row. - tuple := &querypb.Value{ - Type: querypb.Type_TUPLE, - } - for _, colIdx := range child.Cols { - tuple.Values = append(tuple.Values, - sqltypes.ValueToProto(row[colIdx])) - } - bv.Values = append(bv.Values, tuple) + // Create a tuple from each Row. + tuple := &querypb.Value{ + Type: querypb.Type_TUPLE, + } + for _, colIdx := range child.Cols { + tuple.Values = append(tuple.Values, + sqltypes.ValueToProto(row[colIdx])) } + bv.Values = append(bv.Values, tuple) // Execute the child primitive, and bail out incase of failure. // Since this Primitive is always executed in a transaction, the changes should // be rolled back incase of an error. bindVars[child.BVName] = bv - _, err = vcursor.ExecutePrimitive(ctx, child.Exec, bindVars, wantfields) + + for idx, updateExprBvName := range child.UpdateExprBvNames { + bindVars[updateExprBvName] = sqltypes.ValueBindVariable(row[child.UpdateExprCols[idx]]) + } + _, err := vcursor.ExecutePrimitive(ctx, child.Exec, bindVars, wantfields) if err != nil { - return nil, err + return err } delete(bindVars, child.BVName) + for _, updateExprBvName := range child.UpdateExprBvNames { + delete(bindVars, updateExprBvName) + } } - - // All the children are modified successfully, we can now execute the Parent Primitive. - return vcursor.ExecutePrimitive(ctx, fkc.Parent, bindVars, wantfields) + return nil } // TryStreamExecute implements the Primitive interface. @@ -124,6 +189,7 @@ func (fkc *FkCascade) TryStreamExecute(ctx context.Context, vcursor VCursor, bin }) } + // TODO: add execution for non-literal updates. // Execute the Selection primitive to find the rows that are going to modified. // This will be used to find the rows that need modification on the children. err := vcursor.StreamExecutePrimitive(ctx, fkc.Selection, bindVars, wantfields, func(result *sqltypes.Result) error { @@ -177,9 +243,12 @@ func (fkc *FkCascade) Inputs() ([]Primitive, []map[string]any) { }) for idx, child := range fkc.Children { inputsMap = append(inputsMap, map[string]any{ - inputName: fmt.Sprintf("CascadeChild-%d", idx+1), - "BvName": child.BVName, - "Cols": child.Cols, + inputName: fmt.Sprintf("CascadeChild-%d", idx+1), + "BvName": child.BVName, + "Cols": child.Cols, + "UpdateExprBvNames": child.UpdateExprBvNames, + "UpdateExprCols": child.UpdateExprCols, + "CompExprCols": child.CompExprCols, }) inputs = append(inputs, child.Exec) } diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 381aeb704c5..0b5dca6a33d 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -139,9 +139,12 @@ func transformFkCascade(ctx *plancontext.PlanningContext, fkc *operators.FkCasca childEngine := childLP.Primitive() children = append(children, &engine.FkChild{ - BVName: child.BVName, - Cols: child.Cols, - Exec: childEngine, + BVName: child.BVName, + Cols: child.Cols, + UpdateExprBvNames: child.UpdateExprBvNames, + UpdateExprCols: child.UpdateExprCols, + CompExprCols: child.CompExprCols, + Exec: childEngine, }) } diff --git a/go/vt/vtgate/planbuilder/operators/ast_to_op.go b/go/vt/vtgate/planbuilder/operators/ast_to_op.go index 51a29c654d5..6fef308c592 100644 --- a/go/vt/vtgate/planbuilder/operators/ast_to_op.go +++ b/go/vt/vtgate/planbuilder/operators/ast_to_op.go @@ -28,6 +28,7 @@ import ( ) const foreignKeyConstraintValues = "fkc_vals" +const foreignKeyUpdateExpr = "fkc_upd" // translateQueryToOp creates an operator tree that represents the input SELECT or UNION query func translateQueryToOp(ctx *plancontext.PlanningContext, selStmt sqlparser.Statement) (op ops.Operator, err error) { diff --git a/go/vt/vtgate/planbuilder/operators/fk_cascade.go b/go/vt/vtgate/planbuilder/operators/fk_cascade.go index 90c797d55e8..cc764fbcab1 100644 --- a/go/vt/vtgate/planbuilder/operators/fk_cascade.go +++ b/go/vt/vtgate/planbuilder/operators/fk_cascade.go @@ -25,9 +25,12 @@ import ( // FkChild is used to represent a foreign key child table operation type FkChild struct { - BVName string - Cols []int // indexes - Op ops.Operator + BVName string + Cols []int // indexes + UpdateExprBvNames []string + UpdateExprCols []int + CompExprCols []int + Op ops.Operator noColumns noPredicates @@ -88,9 +91,12 @@ func (fkc *FkCascade) Clone(inputs []ops.Operator) ops.Operator { } newFkc.Children = append(newFkc.Children, &FkChild{ - BVName: fkc.Children[idx-2].BVName, - Cols: slices.Clone(fkc.Children[idx-2].Cols), - Op: operator, + BVName: fkc.Children[idx-2].BVName, + Cols: slices.Clone(fkc.Children[idx-2].Cols), + UpdateExprCols: slices.Clone(fkc.Children[idx-2].UpdateExprCols), + UpdateExprBvNames: slices.Clone(fkc.Children[idx-2].UpdateExprBvNames), + CompExprCols: slices.Clone(fkc.Children[idx-2].CompExprCols), + Op: operator, }) } return newFkc diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index bdfba5def10..722808c9e3b 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -203,11 +203,6 @@ func createUpdateOperator(ctx *plancontext.PlanningContext, updStmt *sqlparser.U } func buildFkOperator(ctx *plancontext.PlanningContext, updOp ops.Operator, updClone *sqlparser.Update, parentFks []vindexes.ParentFKInfo, childFks []vindexes.ChildFKInfo, updatedTable *vindexes.Table) (ops.Operator, error) { - // We only support simple expressions in update queries for foreign key handling. - if isNonLiteral(updClone.Exprs, parentFks, childFks) { - return nil, vterrors.VT12001("update expression with non-literal values with foreign key constraints") - } - restrictChildFks, cascadeChildFks := splitChildFks(childFks) op, err := createFKCascadeOp(ctx, updOp, updClone, cascadeChildFks, updatedTable) @@ -218,7 +213,7 @@ func buildFkOperator(ctx *plancontext.PlanningContext, updOp ops.Operator, updCl return createFKVerifyOp(ctx, op, updClone, parentFks, restrictChildFks) } -func isNonLiteral(updExprs sqlparser.UpdateExprs, parentFks []vindexes.ParentFKInfo, childFks []vindexes.ChildFKInfo) bool { +func hasNonLiteral(updExprs sqlparser.UpdateExprs, parentFks []vindexes.ParentFKInfo, childFks []vindexes.ChildFKInfo) bool { for _, updateExpr := range updExprs { if sqlparser.IsLiteral(updateExpr.Expr) { continue @@ -270,11 +265,33 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, return nil, vterrors.VT13001("ON UPDATE RESTRICT foreign keys should already be filtered") } + nonLiteralUpdate := hasNonLiteral(updStmt.Exprs, nil, []vindexes.ChildFKInfo{fk}) + // We need to select all the parent columns for the foreign key constraint, to use in the update of the child table. - var offsets []int - offsets, selectExprs = addColumns(ctx, fk.ParentColumns, selectExprs) + var selectOffsets []int + selectOffsets, selectExprs = addColumns(ctx, fk.ParentColumns, selectExprs) + + // If we are updating a foreign key column to a non-literal value, then we need to get the updated value and + // whether it is different from the current value or not as well. + var updatedOffsets []int + var compOffsets []int + if nonLiteralUpdate { + // TODO: may only store non-literal update exprs OR store non-literal info along with update expr. + updExprs := ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] + for _, updExpr := range updExprs { + if sqlparser.IsLiteral(updExpr.Expr) { + updatedOffsets = append(updatedOffsets, -1) + compOffsets = append(compOffsets, -1) + continue + } + var updateExprOffset, compExprOffset int + updateExprOffset, compExprOffset, selectExprs = addUpdExprToSelect(ctx, updExpr, selectExprs) + updatedOffsets = append(updatedOffsets, updateExprOffset) + compOffsets = append(compOffsets, compExprOffset) + } + } - fkChild, err := createFkChildForUpdate(ctx, fk, updStmt, offsets, updatedTable) + fkChild, err := createFkChildForUpdate(ctx, fk, updStmt, selectOffsets, updatedOffsets, compOffsets, updatedTable) if err != nil { return nil, err } @@ -315,8 +332,26 @@ func addColumns(ctx *plancontext.PlanningContext, columns sqlparser.Columns, exp return offsets, selectExprs } +func addUpdExprToSelect(ctx *plancontext.PlanningContext, updExpr *sqlparser.UpdateExpr, exprs []sqlparser.SelectExpr) (int, int, []sqlparser.SelectExpr) { + var updateExprOffset, compExprOffset int + updateExprOffset, exprs = addExprToSelect(ctx, updExpr.Expr, exprs) + compExpr := sqlparser.NewComparisonExpr(sqlparser.NotEqualOp, updExpr.Name, updExpr.Expr, nil) + compExprOffset, exprs = addExprToSelect(ctx, compExpr, exprs) + return updateExprOffset, compExprOffset, exprs +} + +func addExprToSelect(ctx *plancontext.PlanningContext, expr sqlparser.Expr, exprs []sqlparser.SelectExpr) (int, []sqlparser.SelectExpr) { + for idx, selectExpr := range exprs { + if ctx.SemTable.EqualsExpr(selectExpr.(*sqlparser.AliasedExpr).Expr, expr) { + return idx, exprs + } + } + offset := len(exprs) + return offset, append(exprs, aeWrap(expr)) +} + // createFkChildForUpdate creates the update query operator for the child table based on the foreign key constraints. -func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, cols []int, updatedTable *vindexes.Table) (*FkChild, error) { +func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, selectOffsets, updatedOffsets, compOffsets []int, updatedTable *vindexes.Table) (*FkChild, error) { // Create a ValTuple of child column names var valTuple sqlparser.ValTuple for _, column := range fk.ChildColumns { @@ -329,13 +364,25 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF compExpr := sqlparser.NewComparisonExpr(sqlparser.InOp, valTuple, sqlparser.NewListArg(bvName), nil) var childWhereExpr sqlparser.Expr = compExpr + var updateExprBvNames []string + if len(updatedOffsets) > 0 { + for _, updateOffset := range updatedOffsets { + if updateOffset == -1 { + updateExprBvNames = append(updateExprBvNames, "") + continue + } + updateBvName := ctx.ReservedVars.ReserveVariable(foreignKeyUpdateExpr) + updateExprBvNames = append(updateExprBvNames, updateBvName) + } + } + var childOp ops.Operator var err error switch fk.OnUpdate { case sqlparser.Cascade: - childOp, err = buildChildUpdOpForCascade(ctx, fk, updStmt, childWhereExpr, updatedTable) + childOp, err = buildChildUpdOpForCascade(ctx, fk, updStmt, childWhereExpr, updateExprBvNames, updatedTable) case sqlparser.SetNull: - childOp, err = buildChildUpdOpForSetNull(ctx, fk, updStmt, childWhereExpr) + childOp, err = buildChildUpdOpForSetNull(ctx, fk, updStmt, childWhereExpr, updateExprBvNames, updatedTable) case sqlparser.SetDefault: return nil, vterrors.VT09016() } @@ -343,22 +390,41 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF return nil, err } + updatedOffsets, compOffsets, updateExprBvNames = compressUpdateOffsets(updatedOffsets, compOffsets, updateExprBvNames) + return &FkChild{ - BVName: bvName, - Cols: cols, - Op: childOp, + BVName: bvName, + Cols: selectOffsets, + Op: childOp, + UpdateExprBvNames: updateExprBvNames, + UpdateExprCols: updatedOffsets, + CompExprCols: compOffsets, }, nil } +func compressUpdateOffsets(updatedOffsets []int, compOffsets []int, updateExprBvNames []string) ([]int, []int, []string) { + var newUpdatedOffsets, newCompOffsets []int + var newUpdateExprBvNames []string + for idx, updateOffset := range updatedOffsets { + if updateOffset == -1 { + continue + } + newUpdatedOffsets = append(newUpdatedOffsets, updateOffset) + newCompOffsets = append(newCompOffsets, compOffsets[idx]) + newUpdateExprBvNames = append(newUpdateExprBvNames, updateExprBvNames[idx]) + } + return newUpdatedOffsets, newCompOffsets, newUpdateExprBvNames +} + // buildChildUpdOpForCascade builds the child update statement operator for the CASCADE type foreign key constraint. // The query looks like this - // // `UPDATE SET WHERE IN ()` -func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, childWhereExpr sqlparser.Expr, updatedTable *vindexes.Table) (ops.Operator, error) { +func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, childWhereExpr sqlparser.Expr, updatedExprBvNames []string, updatedTable *vindexes.Table) (ops.Operator, error) { // The update expressions are the same as the update expressions in the parent update query // with the column names replaced with the child column names. var childUpdateExprs sqlparser.UpdateExprs - for _, updateExpr := range updStmt.Exprs { + for idx, updateExpr := range ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] { colIdx := fk.ParentColumns.FindColumn(updateExpr.Name.Name) if colIdx == -1 { continue @@ -366,9 +432,13 @@ func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.Chi // The where condition is the same as the comparison expression above // with the column names replaced with the child column names. + childUpdateExpr := updateExpr.Expr + if len(updatedExprBvNames) > 0 && updatedExprBvNames[idx] != "" { + childUpdateExpr = sqlparser.NewArgument(updatedExprBvNames[idx]) + } childUpdateExprs = append(childUpdateExprs, &sqlparser.UpdateExpr{ Name: sqlparser.NewColName(fk.ChildColumns[colIdx].String()), - Expr: updateExpr.Expr, + Expr: childUpdateExpr, }) } // Because we could be updating the child to a non-null value, @@ -395,7 +465,7 @@ func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.Chi // `UPDATE SET // WHERE IN () // [AND ({ IS NULL OR}... NOT IN ())]` -func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, childWhereExpr sqlparser.Expr) (ops.Operator, error) { +func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, childWhereExpr sqlparser.Expr, updateExprBvNames []string, updatedTable *vindexes.Table) (ops.Operator, error) { // For the SET NULL type constraint, we need to set all the child columns to NULL. var childUpdateExprs sqlparser.UpdateExprs for _, column := range fk.ChildColumns { @@ -414,7 +484,7 @@ func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.Chi // For example, if we are setting `update parent cola = :v1 and colb = :v2`, then on the child, the where condition would look something like this - // `:v1 IS NULL OR :v2 IS NULL OR (child_cola, child_colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL). - compExpr := nullSafeNotInComparison(updStmt.Exprs, fk) + compExpr := nullSafeNotInComparison(ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)], fk, updateExprBvNames) if compExpr != nil { childWhereExpr = &sqlparser.AndExpr{ Left: childWhereExpr, @@ -435,6 +505,11 @@ func createFKVerifyOp(ctx *plancontext.PlanningContext, childOp ops.Operator, up return childOp, nil } + // We only support simple expressions in update queries for foreign key verification. + if hasNonLiteral(updStmt.Exprs, parentFks, restrictChildFks) { + return nil, vterrors.VT12001("update expression with non-literal values with foreign key constraints") + } + var Verify []*VerifyOp // This validates that new values exists on the parent table. for _, fk := range parentFks { @@ -595,7 +670,7 @@ func createFkVerifyOpForChildFKForUpdate(ctx *plancontext.PlanningContext, updSt // For example, if we are setting `update child cola = :v1 and colb = :v2`, then on the parent, the where condition would look something like this - // `:v1 IS NULL OR :v2 IS NULL OR (cola, colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL). - compExpr := nullSafeNotInComparison(updStmt.Exprs, cFk) + compExpr := nullSafeNotInComparison(updStmt.Exprs, cFk, nil) if compExpr != nil { whereCond = sqlparser.AndExpressions(whereCond, compExpr) } @@ -619,15 +694,19 @@ func createFkVerifyOpForChildFKForUpdate(ctx *plancontext.PlanningContext, updSt // `:v1 IS NULL OR :v2 IS NULL OR (cola, colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL) // This expression is used in cascading SET NULLs and in verifying whether an update should be restricted. -func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.ChildFKInfo) sqlparser.Expr { +func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.ChildFKInfo, updatedExprBvNames []string) sqlparser.Expr { var updateValues sqlparser.ValTuple - for _, updateExpr := range updateExprs { + for idx, updateExpr := range updateExprs { colIdx := cFk.ParentColumns.FindColumn(updateExpr.Name.Name) if colIdx >= 0 { if sqlparser.IsNull(updateExpr.Expr) { return nil } - updateValues = append(updateValues, updateExpr.Expr) + childUpdateExpr := updateExpr.Expr + if len(updatedExprBvNames) > 0 && updatedExprBvNames[idx] != "" { + childUpdateExpr = sqlparser.NewArgument(updatedExprBvNames[idx]) + } + updateValues = append(updateValues, childUpdateExpr) } } // Create a ValTuple of child column names diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index c689803f258..0b803679c6b 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -791,12 +791,203 @@ { "comment": "update in a table with non-literal value - set null fail due to child update where condition", "query": "update u_tbl2 set m = 2, col2 = col1 + 'bar' where id = 1", - "plan": "VT12001: unsupported: update expression with non-literal values with foreign key constraints" + "plan": { + "QueryType": "UPDATE", + "Original": "update u_tbl2 set m = 2, col2 = col1 + 'bar' where id = 1", + "Instructions": { + "OperatorType": "FkCascade", + "Inputs": [ + { + "InputName": "Selection", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select col2, col1 + 'bar', col2 != col1 + 'bar' from u_tbl2 where 1 != 1", + "Query": "select col2, col1 + 'bar', col2 != col1 + 'bar' from u_tbl2 where id = 1 for update", + "Table": "u_tbl2" + }, + { + "InputName": "CascadeChild-1", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "BvName": "fkc_vals", + "Cols": [ + 0 + ], + "Query": "update u_tbl3 set col3 = null where (col3) in ::fkc_vals and (:fkc_upd is null or (u_tbl3.col3) not in ((:fkc_upd)))", + "Table": "u_tbl3" + }, + { + "InputName": "Parent", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update u_tbl2 set m = 2, col2 = col1 + 'bar' where id = 1", + "Table": "u_tbl2" + } + ] + }, + "TablesUsed": [ + "unsharded_fk_allow.u_tbl2", + "unsharded_fk_allow.u_tbl3" + ] + } }, { "comment": "update in a table with non-literal value - with cascade fail as the cascade value is not known", "query": "update u_tbl1 set m = 2, col1 = x + 'bar' where id = 1", - "plan": "VT12001: unsupported: update expression with non-literal values with foreign key constraints" + "plan": { + "QueryType": "UPDATE", + "Original": "update u_tbl1 set m = 2, col1 = x + 'bar' where id = 1", + "Instructions": { + "OperatorType": "FkCascade", + "Inputs": [ + { + "InputName": "Selection", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select col1, x + 'bar', col1 != x + 'bar' from u_tbl1 where 1 != 1", + "Query": "select col1, x + 'bar', col1 != x + 'bar' from u_tbl1 where id = 1 for update", + "Table": "u_tbl1" + }, + { + "InputName": "CascadeChild-1", + "OperatorType": "FkCascade", + "BvName": "fkc_vals", + "Cols": [ + 0 + ], + "Inputs": [ + { + "InputName": "Selection", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select col2 from u_tbl2 where 1 != 1", + "Query": "select col2 from u_tbl2 where (col2) in ::fkc_vals for update", + "Table": "u_tbl2" + }, + { + "InputName": "CascadeChild-1", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "BvName": "fkc_vals1", + "Cols": [ + 0 + ], + "Query": "update u_tbl3 set col3 = null where (col3) in ::fkc_vals1 and (:fkc_upd is null or (u_tbl3.col3) not in ((:fkc_upd)))", + "Table": "u_tbl3" + }, + { + "InputName": "Parent", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_tbl2 set col2 = :fkc_upd where (col2) in ::fkc_vals", + "Table": "u_tbl2" + } + ] + }, + { + "InputName": "CascadeChild-2", + "OperatorType": "FkCascade", + "BvName": "fkc_vals2", + "Cols": [ + 0 + ], + "Inputs": [ + { + "InputName": "Selection", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select col9 from u_tbl9 where 1 != 1", + "Query": "select col9 from u_tbl9 where (col9) in ::fkc_vals2 and (:fkc_upd1 is null or (u_tbl9.col9) not in ((:fkc_upd1))) for update", + "Table": "u_tbl9" + }, + { + "InputName": "CascadeChild-1", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "BvName": "fkc_vals3", + "Cols": [ + 0 + ], + "Query": "update u_tbl8 set col8 = null where (col8) in ::fkc_vals3", + "Table": "u_tbl8" + }, + { + "InputName": "Parent", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update u_tbl9 set col9 = null where (col9) in ::fkc_vals2 and (:fkc_upd1 is null or (u_tbl9.col9) not in ((:fkc_upd1)))", + "Table": "u_tbl9" + } + ] + }, + { + "InputName": "Parent", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update u_tbl1 set m = 2, col1 = x + 'bar' where id = 1", + "Table": "u_tbl1" + } + ] + }, + "TablesUsed": [ + "unsharded_fk_allow.u_tbl1", + "unsharded_fk_allow.u_tbl2", + "unsharded_fk_allow.u_tbl3", + "unsharded_fk_allow.u_tbl8", + "unsharded_fk_allow.u_tbl9" + ] + } }, { "comment": "update in a table with set null, non-literal value on non-foreign key column - allowed", From c0c9ae6e73ce85a279fdb2e3080cbe1d93d17b36 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Oct 2023 12:09:46 +0530 Subject: [PATCH 05/43] test: update tests and only print values if they aren't empty Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/fk_cascade.go | 19 +++++++----- .../testdata/foreignkey_cases.json | 29 ++++++++++++++++++- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index c872c7f401e..d8e1c91ae32 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -242,14 +242,17 @@ func (fkc *FkCascade) Inputs() ([]Primitive, []map[string]any) { inputName: "Selection", }) for idx, child := range fkc.Children { - inputsMap = append(inputsMap, map[string]any{ - inputName: fmt.Sprintf("CascadeChild-%d", idx+1), - "BvName": child.BVName, - "Cols": child.Cols, - "UpdateExprBvNames": child.UpdateExprBvNames, - "UpdateExprCols": child.UpdateExprCols, - "CompExprCols": child.CompExprCols, - }) + childInfoMap := map[string]any{ + inputName: fmt.Sprintf("CascadeChild-%d", idx+1), + "BvName": child.BVName, + "Cols": child.Cols, + } + if len(child.UpdateExprBvNames) > 0 { + childInfoMap["UpdateExprBvNames"] = child.UpdateExprBvNames + childInfoMap["UpdateExprCols"] = child.UpdateExprCols + childInfoMap["CompExprCols"] = child.CompExprCols + } + inputsMap = append(inputsMap, childInfoMap) inputs = append(inputs, child.Exec) } inputs = append(inputs, fkc.Parent) diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index 0b803679c6b..6d534151729 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -822,8 +822,17 @@ "Cols": [ 0 ], + "CompExprCols": [ + 2 + ], "Query": "update u_tbl3 set col3 = null where (col3) in ::fkc_vals and (:fkc_upd is null or (u_tbl3.col3) not in ((:fkc_upd)))", - "Table": "u_tbl3" + "Table": "u_tbl3", + "UpdateExprBvNames": [ + "fkc_upd" + ], + "UpdateExprCols": [ + 1 + ] }, { "InputName": "Parent", @@ -873,6 +882,15 @@ "Cols": [ 0 ], + "CompExprCols": [ + 2 + ], + "UpdateExprBvNames": [ + "fkc_upd" + ], + "UpdateExprCols": [ + 1 + ], "Inputs": [ { "InputName": "Selection", @@ -923,6 +941,15 @@ "Cols": [ 0 ], + "CompExprCols": [ + 2 + ], + "UpdateExprBvNames": [ + "fkc_upd1" + ], + "UpdateExprCols": [ + 1 + ], "Inputs": [ { "InputName": "Selection", From ca13490f6f75eac467bb955e5dfd6dee3af0e69a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Oct 2023 15:16:52 +0530 Subject: [PATCH 06/43] feat: handle nils in comperator expression Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/fk_cascade.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index d8e1c91ae32..e7eeebfdba7 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -132,6 +132,9 @@ func (fkc *FkCascade) executeNonLiteralUpdateFkChild(ctx context.Context, vcurso for _, row := range selectionRes.Rows { skipRow := true for _, colIdx := range child.CompExprCols { + if row[colIdx].IsNull() { + continue + } hasChanged, err := row[colIdx].ToBool() if err != nil { return err From a68c8d6d0fd3758850e61d6c499f3b1d087b1052 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Oct 2023 15:18:31 +0530 Subject: [PATCH 07/43] test: add a failing test Signed-off-by: Manan Gupta --- .../vtgate/foreignkey/fk_fuzz_test.go | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go index c645aba98f0..0dbdd8cd8e1 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go @@ -643,6 +643,54 @@ func TestFkFuzzTest(t *testing.T) { } } +// TestFkOneCase is for testing a specific set of queries. On the CI this test won't run since we'll keep the queries empty. +func TestFkOneCase(t *testing.T) { + queries := []string{ + "insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "update fk_t10 set col = id + 3 order by id desc", + } + if len(queries) == 0 { + t.Skip("No queries to test") + } + // Wait for schema-tracking to be complete. + waitForSchemaTrackingForFkTables(t) + // Remove all the foreign key constraints for all the replicas. + // We can then verify that the replica, and the primary have the same data, to ensure + // that none of the queries ever lead to cascades/updates on MySQL level. + for _, ks := range []string{shardedKs, unshardedKs} { + replicas := getReplicaTablets(ks) + for _, replica := range replicas { + removeAllForeignKeyConstraints(t, replica, ks) + } + } + + mcmp, closer := start(t) + defer closer() + _ = utils.Exec(t, mcmp.VtConn, "use `uks`") + + // Ensure that the Vitess database is originally empty + ensureDatabaseState(t, mcmp.VtConn, true) + ensureDatabaseState(t, mcmp.MySQLConn, true) + + for _, query := range queries { + _, _ = mcmp.ExecAllowAndCompareError(query) + if t.Failed() { + log.Errorf("Query failed - %v", query) + break + } + } + vitessData := collectFkTablesState(mcmp.VtConn) + for idx, table := range fkTables { + log.Errorf("Vitess data for %v -\n%v", table, vitessData[idx].Rows) + } + + // ensure Vitess database has some data. This ensures not all the commands failed. + ensureDatabaseState(t, mcmp.VtConn, false) + // Verify the consistency of the data. + verifyDataIsCorrect(t, mcmp, 1) +} + // ensureDatabaseState ensures that the database is either empty or not. func ensureDatabaseState(t *testing.T, vtconn *mysql.Conn, empty bool) { results := collectFkTablesState(vtconn) From 27a78beb23d6d8976c338c149ffcea0df64d4bfd Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Oct 2023 19:52:37 +0530 Subject: [PATCH 08/43] feat: copy the order by from the update query Signed-off-by: Manan Gupta --- go/vt/vtgate/planbuilder/operators/ast_to_op.go | 2 ++ go/vt/vtgate/planbuilder/operators/delete.go | 2 +- go/vt/vtgate/planbuilder/operators/update.go | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/ast_to_op.go b/go/vt/vtgate/planbuilder/operators/ast_to_op.go index 6fef308c592..8d00e29918a 100644 --- a/go/vt/vtgate/planbuilder/operators/ast_to_op.go +++ b/go/vt/vtgate/planbuilder/operators/ast_to_op.go @@ -398,6 +398,7 @@ func createSelectionOp( selectExprs []sqlparser.SelectExpr, tableExprs sqlparser.TableExprs, where *sqlparser.Where, + orderBy sqlparser.OrderBy, limit *sqlparser.Limit, lock sqlparser.Lock, ) (ops.Operator, error) { @@ -405,6 +406,7 @@ func createSelectionOp( SelectExprs: selectExprs, From: tableExprs, Where: where, + OrderBy: orderBy, Limit: limit, Lock: lock, } diff --git a/go/vt/vtgate/planbuilder/operators/delete.go b/go/vt/vtgate/planbuilder/operators/delete.go index b3a24524018..dd37ed5db01 100644 --- a/go/vt/vtgate/planbuilder/operators/delete.go +++ b/go/vt/vtgate/planbuilder/operators/delete.go @@ -190,7 +190,7 @@ func createFkCascadeOpForDelete(ctx *plancontext.PlanningContext, parentOp ops.O } fkChildren = append(fkChildren, fkChild) } - selectionOp, err := createSelectionOp(ctx, selectExprs, delStmt.TableExprs, delStmt.Where, nil, sqlparser.ForUpdateLock) + selectionOp, err := createSelectionOp(ctx, selectExprs, delStmt.TableExprs, delStmt.Where, nil, nil, sqlparser.ForUpdateLock) if err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 722808c9e3b..4b1f36d2e43 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -298,7 +298,7 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, fkChildren = append(fkChildren, fkChild) } - selectionOp, err := createSelectionOp(ctx, selectExprs, updStmt.TableExprs, updStmt.Where, nil, sqlparser.ForUpdateLock) + selectionOp, err := createSelectionOp(ctx, selectExprs, updStmt.TableExprs, updStmt.Where, updStmt.OrderBy, nil, sqlparser.ForUpdateLock) if err != nil { return nil, err } @@ -616,6 +616,7 @@ func createFkVerifyOpForParentFKForUpdate(ctx *plancontext.PlanningContext, updS sqlparser.NewJoinCondition(joinCond, nil)), }, sqlparser.NewWhere(sqlparser.WhereClause, whereCond), + nil, sqlparser.NewLimitWithoutOffset(1), sqlparser.ShareModeLock) } @@ -685,6 +686,7 @@ func createFkVerifyOpForChildFKForUpdate(ctx *plancontext.PlanningContext, updSt sqlparser.NewJoinCondition(joinCond, nil)), }, sqlparser.NewWhere(sqlparser.WhereClause, whereCond), + nil, sqlparser.NewLimitWithoutOffset(1), sqlparser.ShareModeLock) } From 9b191a57da0cc4215071b29f693e98e73ac91bf8 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 19 Oct 2023 20:51:46 +0530 Subject: [PATCH 09/43] feat: turn off foreign key checks on updates that have foreign key columns being set to non-literal values Signed-off-by: Manan Gupta --- .../vtgate/planbuilder/operators/ast_to_op.go | 5 +- go/vt/vtgate/planbuilder/operators/update.go | 28 +++------ .../plancontext/planning_context.go | 1 + go/vt/vtgate/semantics/analyzer.go | 60 +++++++++++++++---- go/vt/vtgate/semantics/analyzer_test.go | 4 +- go/vt/vtgate/semantics/semantic_state.go | 1 + 6 files changed, 65 insertions(+), 34 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/ast_to_op.go b/go/vt/vtgate/planbuilder/operators/ast_to_op.go index 8d00e29918a..64d9826a80e 100644 --- a/go/vt/vtgate/planbuilder/operators/ast_to_op.go +++ b/go/vt/vtgate/planbuilder/operators/ast_to_op.go @@ -214,7 +214,10 @@ func createOpFromStmt(ctx *plancontext.PlanningContext, stmt sqlparser.Statement // we should augment the semantic analysis to also tell us whether the given query has any cross shard parent foreign keys to validate. // If there are, then we have to run the query with FOREIGN_KEY_CHECKS off because we can't be sure if the DML will succeed on MySQL with the checks on. // So, we should set VerifyAllFKs to true. i.e. we should add `|| ctx.SemTable.RequireForeignKeyChecksOff()` to the below condition. - ctx.VerifyAllFKs = verifyAllFKs + if verifyAllFKs { + // If ctx.VerifyAllFKs is already true we don't want to turn it off. + ctx.VerifyAllFKs = verifyAllFKs + } // From all the parent foreign keys involved, we should remove the one that we need to ignore. err = ctx.SemTable.RemoveParentForeignKey(fkToIgnore) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 4b1f36d2e43..5450e157122 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -176,6 +176,11 @@ func createUpdateOperator(ctx *plancontext.PlanningContext, updStmt *sqlparser.U return nil, vterrors.VT12001("multi shard UPDATE with LIMIT") } + if ctx.SemTable.FKChecksOff { + // We have to run the query with FKChecksOff. + updStmt.Comments = updStmt.Comments.Prepend("/*+ SET_VAR(foreign_key_checks=OFF) */").Parsed() + } + route := &Route{ Source: &Update{ QTable: qt, @@ -213,25 +218,6 @@ func buildFkOperator(ctx *plancontext.PlanningContext, updOp ops.Operator, updCl return createFKVerifyOp(ctx, op, updClone, parentFks, restrictChildFks) } -func hasNonLiteral(updExprs sqlparser.UpdateExprs, parentFks []vindexes.ParentFKInfo, childFks []vindexes.ChildFKInfo) bool { - for _, updateExpr := range updExprs { - if sqlparser.IsLiteral(updateExpr.Expr) { - continue - } - for _, parentFk := range parentFks { - if parentFk.ChildColumns.FindColumn(updateExpr.Name.Name) >= 0 { - return true - } - } - for _, childFk := range childFks { - if childFk.ParentColumns.FindColumn(updateExpr.Name.Name) >= 0 { - return true - } - } - } - return false -} - // splitChildFks splits the child foreign keys into restrict and cascade list as restrict is handled through Verify operator and cascade is handled through Cascade operator. func splitChildFks(fks []vindexes.ChildFKInfo) (restrictChildFks, cascadeChildFks []vindexes.ChildFKInfo) { for _, fk := range fks { @@ -265,7 +251,7 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, return nil, vterrors.VT13001("ON UPDATE RESTRICT foreign keys should already be filtered") } - nonLiteralUpdate := hasNonLiteral(updStmt.Exprs, nil, []vindexes.ChildFKInfo{fk}) + nonLiteralUpdate := semantics.HasNonLiteral(updStmt.Exprs, nil, []vindexes.ChildFKInfo{fk}) // We need to select all the parent columns for the foreign key constraint, to use in the update of the child table. var selectOffsets []int @@ -506,7 +492,7 @@ func createFKVerifyOp(ctx *plancontext.PlanningContext, childOp ops.Operator, up } // We only support simple expressions in update queries for foreign key verification. - if hasNonLiteral(updStmt.Exprs, parentFks, restrictChildFks) { + if semantics.HasNonLiteral(updStmt.Exprs, parentFks, restrictChildFks) { return nil, vterrors.VT12001("update expression with non-literal values with foreign key constraints") } diff --git a/go/vt/vtgate/planbuilder/plancontext/planning_context.go b/go/vt/vtgate/planbuilder/plancontext/planning_context.go index 68ccc95b9fd..e6469a3cd75 100644 --- a/go/vt/vtgate/planbuilder/plancontext/planning_context.go +++ b/go/vt/vtgate/planbuilder/plancontext/planning_context.go @@ -78,6 +78,7 @@ func CreatePlanningContext(stmt sqlparser.Statement, SkipPredicates: map[sqlparser.Expr]any{}, PlannerVersion: version, ReservedArguments: map[sqlparser.Expr]string{}, + VerifyAllFKs: semTable.FKChecksOff, }, nil } diff --git a/go/vt/vtgate/semantics/analyzer.go b/go/vt/vtgate/semantics/analyzer.go index 3204378f56c..b40ee312267 100644 --- a/go/vt/vtgate/semantics/analyzer.go +++ b/go/vt/vtgate/semantics/analyzer.go @@ -107,7 +107,7 @@ func (a *analyzer) newSemTable(statement sqlparser.Statement, coll collations.ID columns[union] = info.exprs } - childFks, parentFks, childFkToUpdExprs, err := a.getInvolvedForeignKeys(statement) + childFks, parentFks, childFkToUpdExprs, fkChecksOff, err := a.getInvolvedForeignKeys(statement) if err != nil { return nil, err } @@ -130,6 +130,7 @@ func (a *analyzer) newSemTable(statement sqlparser.Statement, coll collations.ID childForeignKeysInvolved: childFks, parentForeignKeysInvolved: parentFks, ChildFkToUpdExprs: childFkToUpdExprs, + FKChecksOff: fkChecksOff, }, nil } @@ -321,14 +322,14 @@ func (a *analyzer) noteQuerySignature(node sqlparser.SQLNode) { } // getInvolvedForeignKeys gets the foreign keys that might require taking care off when executing the given statement. -func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs, error) { +func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs, bool, error) { // There are only the DML statements that require any foreign keys handling. switch stmt := statement.(type) { case *sqlparser.Delete: // For DELETE statements, none of the parent foreign keys require handling. // So we collect all the child foreign keys. allChildFks, _, err := a.getAllManagedForeignKeys() - return allChildFks, nil, nil, err + return allChildFks, nil, nil, false, err case *sqlparser.Insert: // For INSERT statements, we have 3 different cases: // 1. REPLACE statement: REPLACE statements are essentially DELETEs and INSERTs rolled into one. @@ -338,29 +339,68 @@ func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[Ta // 3. INSERT with ON DUPLICATE KEY UPDATE: This might trigger an update on the columns specified in the ON DUPLICATE KEY UPDATE clause. allChildFks, allParentFKs, err := a.getAllManagedForeignKeys() if err != nil { - return nil, nil, nil, err + return nil, nil, nil, false, err } if stmt.Action == sqlparser.ReplaceAct { - return allChildFks, allParentFKs, nil, nil + return allChildFks, allParentFKs, nil, false, nil } if len(stmt.OnDup) == 0 { - return nil, allParentFKs, nil, nil + return nil, allParentFKs, nil, false, nil } // If only a certain set of columns are being updated, then there might be some child foreign keys that don't need any consideration since their columns aren't being updated. // So, we filter these child foreign keys out. We can't filter any parent foreign keys because the statement will INSERT a row too, which requires validating all the parent foreign keys. updatedChildFks, _, childFkToUpdExprs := a.filterForeignKeysUsingUpdateExpressions(allChildFks, nil, sqlparser.UpdateExprs(stmt.OnDup)) - return updatedChildFks, allParentFKs, childFkToUpdExprs, nil + return updatedChildFks, allParentFKs, childFkToUpdExprs, false, nil case *sqlparser.Update: // For UPDATE queries we get all the parent and child foreign keys, but we can filter some of them out if the columns that they consist off aren't being updated or are set to NULLs. allChildFks, allParentFks, err := a.getAllManagedForeignKeys() if err != nil { - return nil, nil, nil, err + return nil, nil, nil, false, err } childFks, parentFks, childFkToUpdExprs := a.filterForeignKeysUsingUpdateExpressions(allChildFks, allParentFks, stmt.Exprs) - return childFks, parentFks, childFkToUpdExprs, nil + fkChecksOff := false + if HasNonLiteral(stmt.Exprs, collectParentFksFromMap(parentFks), collectChildFksFromMap(childFks)) { + fkChecksOff = true + } + return childFks, parentFks, childFkToUpdExprs, fkChecksOff, nil default: - return nil, nil, nil, nil + return nil, nil, nil, false, nil + } +} + +func collectParentFksFromMap(parentFkMap map[TableSet][]vindexes.ParentFKInfo) []vindexes.ParentFKInfo { + var parentFks []vindexes.ParentFKInfo + for _, fkInfos := range parentFkMap { + parentFks = append(parentFks, fkInfos...) + } + return parentFks +} + +func collectChildFksFromMap(childFkMap map[TableSet][]vindexes.ChildFKInfo) []vindexes.ChildFKInfo { + var childFks []vindexes.ChildFKInfo + for _, fkInfos := range childFkMap { + childFks = append(childFks, fkInfos...) + } + return childFks +} + +func HasNonLiteral(updExprs sqlparser.UpdateExprs, parentFks []vindexes.ParentFKInfo, childFks []vindexes.ChildFKInfo) bool { + for _, updateExpr := range updExprs { + if sqlparser.IsLiteral(updateExpr.Expr) { + continue + } + for _, parentFk := range parentFks { + if parentFk.ChildColumns.FindColumn(updateExpr.Name.Name) >= 0 { + return true + } + } + for _, childFk := range childFks { + if childFk.ParentColumns.FindColumn(updateExpr.Name.Name) >= 0 { + return true + } + } } + return false } // filterForeignKeysUsingUpdateExpressions filters the child and parent foreign key constraints that don't require any validations/cascades given the updated expressions. diff --git a/go/vt/vtgate/semantics/analyzer_test.go b/go/vt/vtgate/semantics/analyzer_test.go index b31590c09b2..c924a9f739c 100644 --- a/go/vt/vtgate/semantics/analyzer_test.go +++ b/go/vt/vtgate/semantics/analyzer_test.go @@ -1701,7 +1701,7 @@ func TestFilterForeignKeysUsingUpdateExpressions(t *testing.T) { }, si: &FakeSI{ KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_FK_MANAGED, + "ks": vschemapb.Keyspace_managed, }, }, }, @@ -2000,7 +2000,7 @@ func TestGetInvolvedForeignKeys(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - childFks, parentFks, _, err := tt.analyzer.getInvolvedForeignKeys(tt.stmt) + childFks, parentFks, _, _, err := tt.analyzer.getInvolvedForeignKeys(tt.stmt) if tt.expectedErr != "" { require.EqualError(t, err, tt.expectedErr) return diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 16d77ec1080..088a56fc2b5 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -132,6 +132,7 @@ type ( childForeignKeysInvolved map[TableSet][]vindexes.ChildFKInfo parentForeignKeysInvolved map[TableSet][]vindexes.ParentFKInfo ChildFkToUpdExprs map[string]sqlparser.UpdateExprs + FKChecksOff bool } columnName struct { From 8470476cf27c5cd2b8d34730ec297be2352465a7 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 20 Oct 2023 11:37:31 +0530 Subject: [PATCH 10/43] test: refactor tests and permanently add cases for updates with non-literal expressions Signed-off-by: Manan Gupta --- .../vtgate/foreignkey/fk_fuzz_test.go | 132 ------------------ go/test/endtoend/vtgate/foreignkey/fk_test.go | 105 ++++++++++++++ .../endtoend/vtgate/foreignkey/utils_test.go | 84 +++++++++++ 3 files changed, 189 insertions(+), 132 deletions(-) diff --git a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go index 0dbdd8cd8e1..bf3a1ffc68a 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go @@ -28,9 +28,7 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/utils" "vitess.io/vitess/go/vt/log" ) @@ -642,133 +640,3 @@ func TestFkFuzzTest(t *testing.T) { } } } - -// TestFkOneCase is for testing a specific set of queries. On the CI this test won't run since we'll keep the queries empty. -func TestFkOneCase(t *testing.T) { - queries := []string{ - "insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", - "insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", - "update fk_t10 set col = id + 3 order by id desc", - } - if len(queries) == 0 { - t.Skip("No queries to test") - } - // Wait for schema-tracking to be complete. - waitForSchemaTrackingForFkTables(t) - // Remove all the foreign key constraints for all the replicas. - // We can then verify that the replica, and the primary have the same data, to ensure - // that none of the queries ever lead to cascades/updates on MySQL level. - for _, ks := range []string{shardedKs, unshardedKs} { - replicas := getReplicaTablets(ks) - for _, replica := range replicas { - removeAllForeignKeyConstraints(t, replica, ks) - } - } - - mcmp, closer := start(t) - defer closer() - _ = utils.Exec(t, mcmp.VtConn, "use `uks`") - - // Ensure that the Vitess database is originally empty - ensureDatabaseState(t, mcmp.VtConn, true) - ensureDatabaseState(t, mcmp.MySQLConn, true) - - for _, query := range queries { - _, _ = mcmp.ExecAllowAndCompareError(query) - if t.Failed() { - log.Errorf("Query failed - %v", query) - break - } - } - vitessData := collectFkTablesState(mcmp.VtConn) - for idx, table := range fkTables { - log.Errorf("Vitess data for %v -\n%v", table, vitessData[idx].Rows) - } - - // ensure Vitess database has some data. This ensures not all the commands failed. - ensureDatabaseState(t, mcmp.VtConn, false) - // Verify the consistency of the data. - verifyDataIsCorrect(t, mcmp, 1) -} - -// ensureDatabaseState ensures that the database is either empty or not. -func ensureDatabaseState(t *testing.T, vtconn *mysql.Conn, empty bool) { - results := collectFkTablesState(vtconn) - isEmpty := true - for _, res := range results { - if len(res.Rows) > 0 { - isEmpty = false - } - } - require.Equal(t, isEmpty, empty) -} - -// verifyDataIsCorrect verifies that the data in MySQL database matches the data in the Vitess database. -func verifyDataIsCorrect(t *testing.T, mcmp utils.MySQLCompare, concurrency int) { - // For single concurrent thread, we run all the queries on both MySQL and Vitess, so we can verify correctness - // by just checking if the data in MySQL and Vitess match. - if concurrency == 1 { - for _, table := range fkTables { - query := fmt.Sprintf("SELECT * FROM %v ORDER BY id", table) - mcmp.Exec(query) - } - } else { - // For higher concurrency, we don't have MySQL data to verify everything is fine, - // so we'll have to do something different. - // We run LEFT JOIN queries on all the parent and child tables linked by foreign keys - // to make sure that nothing is broken in the database. - for _, reference := range fkReferences { - query := fmt.Sprintf("select %v.id from %v left join %v on (%v.col = %v.col) where %v.col is null and %v.col is not null", reference.childTable, reference.childTable, reference.parentTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable) - if isMultiColFkTable(reference.childTable) { - query = fmt.Sprintf("select %v.id from %v left join %v on (%v.cola = %v.cola and %v.colb = %v.colb) where %v.cola is null and %v.cola is not null and %v.colb is not null", reference.childTable, reference.childTable, reference.parentTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable, reference.childTable) - } - res, err := mcmp.VtConn.ExecuteFetch(query, 1000, false) - require.NoError(t, err) - require.Zerof(t, len(res.Rows), "Query %v gave non-empty results", query) - } - } - // We also verify that the results in Primary and Replica table match as is. - for _, keyspace := range clusterInstance.Keyspaces { - for _, shard := range keyspace.Shards { - var primaryTab, replicaTab *cluster.Vttablet - for _, vttablet := range shard.Vttablets { - if vttablet.Type == "primary" { - primaryTab = vttablet - } else { - replicaTab = vttablet - } - } - require.NotNil(t, primaryTab) - require.NotNil(t, replicaTab) - checkReplicationHealthy(t, replicaTab) - cluster.WaitForReplicationPos(t, primaryTab, replicaTab, true, 60.0) - primaryConn, err := utils.GetMySQLConn(primaryTab, fmt.Sprintf("vt_%v", keyspace.Name)) - require.NoError(t, err) - replicaConn, err := utils.GetMySQLConn(replicaTab, fmt.Sprintf("vt_%v", keyspace.Name)) - require.NoError(t, err) - primaryRes := collectFkTablesState(primaryConn) - replicaRes := collectFkTablesState(replicaConn) - verifyDataMatches(t, primaryRes, replicaRes) - } - } -} - -// verifyDataMatches verifies that the two list of results are the same. -func verifyDataMatches(t *testing.T, resOne []*sqltypes.Result, resTwo []*sqltypes.Result) { - require.EqualValues(t, len(resTwo), len(resOne), "Res 1 - %v, Res 2 - %v", resOne, resTwo) - for idx, resultOne := range resOne { - resultTwo := resTwo[idx] - require.True(t, resultOne.Equal(resultTwo), "Data for %v doesn't match\nRows 1\n%v\nRows 2\n%v", fkTables[idx], resultOne.Rows, resultTwo.Rows) - } -} - -// collectFkTablesState collects the data stored in the foreign key tables for the given connection. -func collectFkTablesState(conn *mysql.Conn) []*sqltypes.Result { - var tablesData []*sqltypes.Result - for _, table := range fkTables { - query := fmt.Sprintf("SELECT * FROM %v ORDER BY id", table) - res, _ := conn.ExecuteFetch(query, 10000, true) - tablesData = append(tablesData, res) - } - return tablesData -} diff --git a/go/test/endtoend/vtgate/foreignkey/fk_test.go b/go/test/endtoend/vtgate/foreignkey/fk_test.go index c3be526e584..8588cdf5f63 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_test.go @@ -27,6 +27,7 @@ import ( "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/utils" + "vitess.io/vitess/go/vt/log" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vtgate/vtgateconn" @@ -774,3 +775,107 @@ func TestFkScenarios(t *testing.T) { }) } } + +// TestFkQueries is for testing a specific set of queries one after the other. +func TestFkQueries(t *testing.T) { + // Wait for schema-tracking to be complete. + waitForSchemaTrackingForFkTables(t) + // Remove all the foreign key constraints for all the replicas. + // We can then verify that the replica, and the primary have the same data, to ensure + // that none of the queries ever lead to cascades/updates on MySQL level. + for _, ks := range []string{shardedKs, unshardedKs} { + replicas := getReplicaTablets(ks) + for _, replica := range replicas { + removeAllForeignKeyConstraints(t, replica, ks) + } + } + + testcases := []struct { + name string + queries []string + }{ + { + name: "Non-literal update", + queries: []string{ + "insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "update fk_t10 set col = id + 3", + }, + }, { + name: "Non-literal update with order by", + queries: []string{ + "insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "update fk_t10 set col = id + 3 order by id desc", + }, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + mcmp, closer := start(t) + defer closer() + _ = utils.Exec(t, mcmp.VtConn, "use `uks`") + + // Ensure that the Vitess database is originally empty + ensureDatabaseState(t, mcmp.VtConn, true) + ensureDatabaseState(t, mcmp.MySQLConn, true) + + for _, query := range testcase.queries { + _, _ = mcmp.ExecAllowAndCompareError(query) + if t.Failed() { + break + } + } + + // ensure Vitess database has some data. This ensures not all the commands failed. + ensureDatabaseState(t, mcmp.VtConn, false) + // Verify the consistency of the data. + verifyDataIsCorrect(t, mcmp, 1) + }) + } +} + +// TestFkOneCase is for testing a specific set of queries. On the CI this test won't run since we'll keep the queries empty. +func TestFkOneCase(t *testing.T) { + queries := []string{} + if len(queries) == 0 { + t.Skip("No queries to test") + } + // Wait for schema-tracking to be complete. + waitForSchemaTrackingForFkTables(t) + // Remove all the foreign key constraints for all the replicas. + // We can then verify that the replica, and the primary have the same data, to ensure + // that none of the queries ever lead to cascades/updates on MySQL level. + for _, ks := range []string{shardedKs, unshardedKs} { + replicas := getReplicaTablets(ks) + for _, replica := range replicas { + removeAllForeignKeyConstraints(t, replica, ks) + } + } + + mcmp, closer := start(t) + defer closer() + _ = utils.Exec(t, mcmp.VtConn, "use `uks`") + + // Ensure that the Vitess database is originally empty + ensureDatabaseState(t, mcmp.VtConn, true) + ensureDatabaseState(t, mcmp.MySQLConn, true) + + for _, query := range queries { + _, _ = mcmp.ExecAllowAndCompareError(query) + if t.Failed() { + log.Errorf("Query failed - %v", query) + break + } + } + vitessData := collectFkTablesState(mcmp.VtConn) + for idx, table := range fkTables { + log.Errorf("Vitess data for %v -\n%v", table, vitessData[idx].Rows) + } + + // ensure Vitess database has some data. This ensures not all the commands failed. + ensureDatabaseState(t, mcmp.VtConn, false) + // Verify the consistency of the data. + verifyDataIsCorrect(t, mcmp, 1) +} diff --git a/go/test/endtoend/vtgate/foreignkey/utils_test.go b/go/test/endtoend/vtgate/foreignkey/utils_test.go index d81a40eb90b..bb02fbdbcbe 100644 --- a/go/test/endtoend/vtgate/foreignkey/utils_test.go +++ b/go/test/endtoend/vtgate/foreignkey/utils_test.go @@ -25,6 +25,8 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/endtoend/cluster" "vitess.io/vitess/go/test/endtoend/utils" ) @@ -171,3 +173,85 @@ func compareVitessAndMySQLErrors(t *testing.T, vtErr, mysqlErr error) { out := fmt.Sprintf("Vitess and MySQL are not erroring the same way.\nVitess error: %v\nMySQL error: %v", vtErr, mysqlErr) t.Error(out) } + +// ensureDatabaseState ensures that the database is either empty or not. +func ensureDatabaseState(t *testing.T, vtconn *mysql.Conn, empty bool) { + results := collectFkTablesState(vtconn) + isEmpty := true + for _, res := range results { + if len(res.Rows) > 0 { + isEmpty = false + } + } + require.Equal(t, isEmpty, empty) +} + +// verifyDataIsCorrect verifies that the data in MySQL database matches the data in the Vitess database. +func verifyDataIsCorrect(t *testing.T, mcmp utils.MySQLCompare, concurrency int) { + // For single concurrent thread, we run all the queries on both MySQL and Vitess, so we can verify correctness + // by just checking if the data in MySQL and Vitess match. + if concurrency == 1 { + for _, table := range fkTables { + query := fmt.Sprintf("SELECT * FROM %v ORDER BY id", table) + mcmp.Exec(query) + } + } else { + // For higher concurrency, we don't have MySQL data to verify everything is fine, + // so we'll have to do something different. + // We run LEFT JOIN queries on all the parent and child tables linked by foreign keys + // to make sure that nothing is broken in the database. + for _, reference := range fkReferences { + query := fmt.Sprintf("select %v.id from %v left join %v on (%v.col = %v.col) where %v.col is null and %v.col is not null", reference.childTable, reference.childTable, reference.parentTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable) + if isMultiColFkTable(reference.childTable) { + query = fmt.Sprintf("select %v.id from %v left join %v on (%v.cola = %v.cola and %v.colb = %v.colb) where %v.cola is null and %v.cola is not null and %v.colb is not null", reference.childTable, reference.childTable, reference.parentTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable, reference.parentTable, reference.childTable, reference.childTable) + } + res, err := mcmp.VtConn.ExecuteFetch(query, 1000, false) + require.NoError(t, err) + require.Zerof(t, len(res.Rows), "Query %v gave non-empty results", query) + } + } + // We also verify that the results in Primary and Replica table match as is. + for _, keyspace := range clusterInstance.Keyspaces { + for _, shard := range keyspace.Shards { + var primaryTab, replicaTab *cluster.Vttablet + for _, vttablet := range shard.Vttablets { + if vttablet.Type == "primary" { + primaryTab = vttablet + } else { + replicaTab = vttablet + } + } + require.NotNil(t, primaryTab) + require.NotNil(t, replicaTab) + checkReplicationHealthy(t, replicaTab) + cluster.WaitForReplicationPos(t, primaryTab, replicaTab, true, 60.0) + primaryConn, err := utils.GetMySQLConn(primaryTab, fmt.Sprintf("vt_%v", keyspace.Name)) + require.NoError(t, err) + replicaConn, err := utils.GetMySQLConn(replicaTab, fmt.Sprintf("vt_%v", keyspace.Name)) + require.NoError(t, err) + primaryRes := collectFkTablesState(primaryConn) + replicaRes := collectFkTablesState(replicaConn) + verifyDataMatches(t, primaryRes, replicaRes) + } + } +} + +// verifyDataMatches verifies that the two list of results are the same. +func verifyDataMatches(t *testing.T, resOne []*sqltypes.Result, resTwo []*sqltypes.Result) { + require.EqualValues(t, len(resTwo), len(resOne), "Res 1 - %v, Res 2 - %v", resOne, resTwo) + for idx, resultOne := range resOne { + resultTwo := resTwo[idx] + require.True(t, resultOne.Equal(resultTwo), "Data for %v doesn't match\nRows 1\n%v\nRows 2\n%v", fkTables[idx], resultOne.Rows, resultTwo.Rows) + } +} + +// collectFkTablesState collects the data stored in the foreign key tables for the given connection. +func collectFkTablesState(conn *mysql.Conn) []*sqltypes.Result { + var tablesData []*sqltypes.Result + for _, table := range fkTables { + query := fmt.Sprintf("SELECT * FROM %v ORDER BY id", table) + res, _ := conn.ExecuteFetch(query, 10000, true) + tablesData = append(tablesData, res) + } + return tablesData +} From b1432c408402038d37861715251439a2cf7c4fb8 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 20 Oct 2023 12:20:47 +0530 Subject: [PATCH 11/43] test: udpate tests output Signed-off-by: Manan Gupta --- .../testdata/foreignkey_cases.json | 65 +------------------ 1 file changed, 2 insertions(+), 63 deletions(-) diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index 6d534151729..16b641e58d2 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -791,68 +791,7 @@ { "comment": "update in a table with non-literal value - set null fail due to child update where condition", "query": "update u_tbl2 set m = 2, col2 = col1 + 'bar' where id = 1", - "plan": { - "QueryType": "UPDATE", - "Original": "update u_tbl2 set m = 2, col2 = col1 + 'bar' where id = 1", - "Instructions": { - "OperatorType": "FkCascade", - "Inputs": [ - { - "InputName": "Selection", - "OperatorType": "Route", - "Variant": "Unsharded", - "Keyspace": { - "Name": "unsharded_fk_allow", - "Sharded": false - }, - "FieldQuery": "select col2, col1 + 'bar', col2 != col1 + 'bar' from u_tbl2 where 1 != 1", - "Query": "select col2, col1 + 'bar', col2 != col1 + 'bar' from u_tbl2 where id = 1 for update", - "Table": "u_tbl2" - }, - { - "InputName": "CascadeChild-1", - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "unsharded_fk_allow", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "BvName": "fkc_vals", - "Cols": [ - 0 - ], - "CompExprCols": [ - 2 - ], - "Query": "update u_tbl3 set col3 = null where (col3) in ::fkc_vals and (:fkc_upd is null or (u_tbl3.col3) not in ((:fkc_upd)))", - "Table": "u_tbl3", - "UpdateExprBvNames": [ - "fkc_upd" - ], - "UpdateExprCols": [ - 1 - ] - }, - { - "InputName": "Parent", - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "unsharded_fk_allow", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "update u_tbl2 set m = 2, col2 = col1 + 'bar' where id = 1", - "Table": "u_tbl2" - } - ] - }, - "TablesUsed": [ - "unsharded_fk_allow.u_tbl2", - "unsharded_fk_allow.u_tbl3" - ] - } + "plan": "VT12001: unsupported: update expression with non-literal values with foreign key constraints" }, { "comment": "update in a table with non-literal value - with cascade fail as the cascade value is not known", @@ -1002,7 +941,7 @@ "Sharded": false }, "TargetTabletType": "PRIMARY", - "Query": "update u_tbl1 set m = 2, col1 = x + 'bar' where id = 1", + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_tbl1 set m = 2, col1 = x + 'bar' where id = 1", "Table": "u_tbl1" } ] From fd79f830c120a0e29c053089fdaba957571f629e Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 20 Oct 2023 12:22:58 +0530 Subject: [PATCH 12/43] feat: add AND rewriter to drop some unnecessary predicates Signed-off-by: Manan Gupta --- .../testdata/postprocess_cases.json | 4 +- go/vt/vtgate/semantics/early_rewriter.go | 41 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json index 3560b0f323f..f1eee18b5d8 100644 --- a/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/postprocess_cases.json @@ -88,7 +88,7 @@ "Sharded": true }, "FieldQuery": "select `user`.col1 as a, `user`.col2 from `user` where 1 != 1", - "Query": "select `user`.col1 as a, `user`.col2 from `user` where `user`.col1 = 1 and `user`.col1 = `user`.col2 and 1 = 1", + "Query": "select `user`.col1 as a, `user`.col2 from `user` where `user`.col1 = 1 and `user`.col1 = `user`.col2", "Table": "`user`" }, { @@ -99,7 +99,7 @@ "Sharded": true }, "FieldQuery": "select user_extra.col3 from user_extra where 1 != 1", - "Query": "select user_extra.col3 from user_extra where user_extra.col3 = 1 and 1 = 1", + "Query": "select user_extra.col3 from user_extra where user_extra.col3 = 1", "Table": "user_extra" } ] diff --git a/go/vt/vtgate/semantics/early_rewriter.go b/go/vt/vtgate/semantics/early_rewriter.go index d11d12023c4..ae433465998 100644 --- a/go/vt/vtgate/semantics/early_rewriter.go +++ b/go/vt/vtgate/semantics/early_rewriter.go @@ -47,6 +47,8 @@ func (r *earlyRewriter) down(cursor *sqlparser.Cursor) error { handleOrderBy(r, cursor, node) case *sqlparser.OrExpr: rewriteOrExpr(cursor, node) + case *sqlparser.AndExpr: + rewriteAndExpr(cursor, node) case *sqlparser.NotExpr: rewriteNotExpr(cursor, node) case sqlparser.GroupBy: @@ -138,6 +140,45 @@ func rewriteOrExpr(cursor *sqlparser.Cursor, node *sqlparser.OrExpr) { } } +// rewriteAndExpr rewrites AND expressions when either side is TRUE. +func rewriteAndExpr(cursor *sqlparser.Cursor, node *sqlparser.AndExpr) { + newNode := rewriteAndTrue(*node) + if newNode != nil { + cursor.ReplaceAndRevisit(newNode) + } +} + +func rewriteAndTrue(andExpr sqlparser.AndExpr) sqlparser.Expr { + // we are looking for the pattern `WHERE c = 1 AND 1 = 1` + isTrue := func(subExpr sqlparser.Expr) bool { + evalEnginePred, err := evalengine.Translate(subExpr, nil) + if err != nil { + return false + } + + env := evalengine.EmptyExpressionEnv() + res, err := env.Evaluate(evalEnginePred) + if err != nil { + return false + } + + boolValue, err := res.Value(collations.Default()).ToBool() + if err != nil { + return false + } + + return boolValue + } + + if isTrue(andExpr.Left) { + return andExpr.Right + } else if isTrue(andExpr.Right) { + return andExpr.Left + } + + return nil +} + // handleLiteral processes literals within the context of ORDER BY expressions. func handleLiteral(r *earlyRewriter, cursor *sqlparser.Cursor, node *sqlparser.Literal) error { newNode, err := r.rewriteOrderByExpr(node) From 465df5d0632b3c6383e5a1df5a9e1d93532953d3 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 20 Oct 2023 12:38:32 +0530 Subject: [PATCH 13/43] feat: add non-literal update parent foreign key verification Signed-off-by: Manan Gupta --- go/vt/vtgate/planbuilder/operators/update.go | 22 +++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 5450e157122..94530e7b699 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -492,7 +492,7 @@ func createFKVerifyOp(ctx *plancontext.PlanningContext, childOp ops.Operator, up } // We only support simple expressions in update queries for foreign key verification. - if semantics.HasNonLiteral(updStmt.Exprs, parentFks, restrictChildFks) { + if semantics.HasNonLiteral(updStmt.Exprs, nil, restrictChildFks) { return nil, vterrors.VT12001("update expression with non-literal values with foreign key constraints") } @@ -528,13 +528,13 @@ func createFKVerifyOp(ctx *plancontext.PlanningContext, childOp ops.Operator, up // Each parent foreign key constraint is verified by an anti join query of the form: // select 1 from child_tbl left join parent_tbl on -// where and and limit 1 +// where and and and limit 1 // E.g: // Child (c1, c2) references Parent (p1, p2) -// update Child set c1 = 1 where id = 1 +// update Child set c1 = c2 + 1 where id = 1 // verify query: -// select 1 from Child left join Parent on Parent.p1 = 1 and Parent.p2 = Child.c2 -// where Parent.p1 is null and Parent.p2 is null and Child.id = 1 +// select 1 from Child left join Parent on Parent.p1 = Child.c2 + 1 and Parent.p2 = Child.c2 +// where Parent.p1 is null and Parent.p2 is null and Child.id = 1 and Child.c2 + 1 is not null // and Child.c2 is not null // limit 1 func createFkVerifyOpForParentFKForUpdate(ctx *plancontext.PlanningContext, updStmt *sqlparser.Update, pFK vindexes.ParentFKInfo) (ops.Operator, error) { @@ -562,7 +562,7 @@ func createFkVerifyOpForParentFKForUpdate(ctx *plancontext.PlanningContext, updS var joinExpr sqlparser.Expr if matchedExpr == nil { predicate = &sqlparser.AndExpr{ - Left: parentIsNullExpr, + Left: predicate, Right: &sqlparser.IsExpr{ Left: sqlparser.NewColNameWithQualifier(pFK.ChildColumns[idx].String(), childTbl), Right: sqlparser.IsNotNullOp, @@ -574,10 +574,18 @@ func createFkVerifyOpForParentFKForUpdate(ctx *plancontext.PlanningContext, updS Right: sqlparser.NewColNameWithQualifier(pFK.ChildColumns[idx].String(), childTbl), } } else { + prefixedMatchExpr := prefixColNames(childTbl, matchedExpr.Expr) joinExpr = &sqlparser.ComparisonExpr{ Operator: sqlparser.EqualOp, Left: sqlparser.NewColNameWithQualifier(pFK.ParentColumns[idx].String(), parentTbl), - Right: prefixColNames(childTbl, matchedExpr.Expr), + Right: prefixedMatchExpr, + } + predicate = &sqlparser.AndExpr{ + Left: predicate, + Right: &sqlparser.IsExpr{ + Left: prefixedMatchExpr, + Right: sqlparser.IsNotNullOp, + }, } } From a1b04e4127d604b509f8497592ab511f3048451a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 20 Oct 2023 13:27:23 +0530 Subject: [PATCH 14/43] feat: augment verification queries to work with non-literal values in updates Signed-off-by: Manan Gupta --- go/test/endtoend/vtgate/foreignkey/fk_test.go | 26 +++ go/vt/vtgate/planbuilder/operators/update.go | 17 +- .../testdata/foreignkey_cases.json | 170 +++++++++++++++++- 3 files changed, 196 insertions(+), 17 deletions(-) diff --git a/go/test/endtoend/vtgate/foreignkey/fk_test.go b/go/test/endtoend/vtgate/foreignkey/fk_test.go index 8588cdf5f63..1fff5d279d8 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_test.go @@ -808,6 +808,32 @@ func TestFkQueries(t *testing.T) { "insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", "update fk_t10 set col = id + 3 order by id desc", }, + }, { + name: "Non-literal update with order by that require parent and child foreign keys verification - success", + queries: []string{ + "insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8)", + "insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t13 (id, col) values (1,1),(2,2)", + "update fk_t11 set col = id + 3 where id >= 3", + }, + }, { + name: "Non-literal update with order by that require parent and child foreign keys verification - parent fails", + queries: []string{ + "insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "update fk_t11 set col = id + 3", + }, + }, { + name: "Non-literal update with order by that require parent and child foreign keys verification - child fails", + queries: []string{ + "insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5),(6,6),(7,7),(8,8)", + "insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t13 (id, col) values (1,1),(2,2)", + "update fk_t11 set col = id + 3", + }, }, } diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 94530e7b699..568243a8c98 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -470,7 +470,7 @@ func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.Chi // For example, if we are setting `update parent cola = :v1 and colb = :v2`, then on the child, the where condition would look something like this - // `:v1 IS NULL OR :v2 IS NULL OR (child_cola, child_colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL). - compExpr := nullSafeNotInComparison(ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)], fk, updateExprBvNames) + compExpr := nullSafeNotInComparison(ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)], fk, updatedTable.GetTableName(), updateExprBvNames) if compExpr != nil { childWhereExpr = &sqlparser.AndExpr{ Left: childWhereExpr, @@ -491,11 +491,6 @@ func createFKVerifyOp(ctx *plancontext.PlanningContext, childOp ops.Operator, up return childOp, nil } - // We only support simple expressions in update queries for foreign key verification. - if semantics.HasNonLiteral(updStmt.Exprs, nil, restrictChildFks) { - return nil, vterrors.VT12001("update expression with non-literal values with foreign key constraints") - } - var Verify []*VerifyOp // This validates that new values exists on the parent table. for _, fk := range parentFks { @@ -619,10 +614,10 @@ func createFkVerifyOpForParentFKForUpdate(ctx *plancontext.PlanningContext, updS // select 1 from child_tbl join parent_tbl on where [AND ({ IS NULL OR}... NOT IN ())] limit 1 // E.g: // Child (c1, c2) references Parent (p1, p2) -// update Parent set p1 = 1 where id = 1 +// update Parent set p1 = col + 1 where id = 1 // verify query: // select 1 from Child join Parent on Parent.p1 = Child.c1 and Parent.p2 = Child.c2 -// where Parent.id = 1 and (1 IS NULL OR (child.c1) NOT IN ((1))) limit 1 +// where Parent.id = 1 and ((Parent.col + 1) IS NULL OR (child.c1) NOT IN ((Parent.col + 1))) limit 1 func createFkVerifyOpForChildFKForUpdate(ctx *plancontext.PlanningContext, updStmt *sqlparser.Update, cFk vindexes.ChildFKInfo) (ops.Operator, error) { // ON UPDATE RESTRICT foreign keys that require validation, should only be allowed in the case where we // are verifying all the FKs on vtgate level. @@ -665,7 +660,7 @@ func createFkVerifyOpForChildFKForUpdate(ctx *plancontext.PlanningContext, updSt // For example, if we are setting `update child cola = :v1 and colb = :v2`, then on the parent, the where condition would look something like this - // `:v1 IS NULL OR :v2 IS NULL OR (cola, colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL). - compExpr := nullSafeNotInComparison(updStmt.Exprs, cFk, nil) + compExpr := nullSafeNotInComparison(updStmt.Exprs, cFk, parentTbl, nil) if compExpr != nil { whereCond = sqlparser.AndExpressions(whereCond, compExpr) } @@ -690,7 +685,7 @@ func createFkVerifyOpForChildFKForUpdate(ctx *plancontext.PlanningContext, updSt // `:v1 IS NULL OR :v2 IS NULL OR (cola, colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL) // This expression is used in cascading SET NULLs and in verifying whether an update should be restricted. -func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.ChildFKInfo, updatedExprBvNames []string) sqlparser.Expr { +func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.ChildFKInfo, parentTbl sqlparser.TableName, updatedExprBvNames []string) sqlparser.Expr { var updateValues sqlparser.ValTuple for idx, updateExpr := range updateExprs { colIdx := cFk.ParentColumns.FindColumn(updateExpr.Name.Name) @@ -698,7 +693,7 @@ func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.Chi if sqlparser.IsNull(updateExpr.Expr) { return nil } - childUpdateExpr := updateExpr.Expr + childUpdateExpr := prefixColNames(parentTbl, updateExpr.Expr) if len(updatedExprBvNames) > 0 && updatedExprBvNames[idx] != "" { childUpdateExpr = sqlparser.NewArgument(updatedExprBvNames[idx]) } diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index 16b641e58d2..a4cc831ad20 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -791,7 +791,87 @@ { "comment": "update in a table with non-literal value - set null fail due to child update where condition", "query": "update u_tbl2 set m = 2, col2 = col1 + 'bar' where id = 1", - "plan": "VT12001: unsupported: update expression with non-literal values with foreign key constraints" + "plan": { + "QueryType": "UPDATE", + "Original": "update u_tbl2 set m = 2, col2 = col1 + 'bar' where id = 1", + "Instructions": { + "OperatorType": "FKVerify", + "Inputs": [ + { + "InputName": "VerifyParent-1", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select 1 from u_tbl2 left join u_tbl1 on u_tbl1.col1 = u_tbl2.col1 + 'bar' where 1 != 1", + "Query": "select 1 from u_tbl2 left join u_tbl1 on u_tbl1.col1 = u_tbl2.col1 + 'bar' where u_tbl2.col1 + 'bar' is not null and u_tbl2.id = 1 and u_tbl1.col1 is null limit 1 lock in share mode", + "Table": "u_tbl1, u_tbl2" + }, + { + "InputName": "PostVerify", + "OperatorType": "FkCascade", + "Inputs": [ + { + "InputName": "Selection", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select col2, u_tbl2.col1 + 'bar', col2 != u_tbl2.col1 + 'bar' from u_tbl2 where 1 != 1", + "Query": "select col2, u_tbl2.col1 + 'bar', col2 != u_tbl2.col1 + 'bar' from u_tbl2 where u_tbl2.id = 1 for update", + "Table": "u_tbl2" + }, + { + "InputName": "CascadeChild-1", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "BvName": "fkc_vals", + "Cols": [ + 0 + ], + "CompExprCols": [ + 2 + ], + "Query": "update u_tbl3 set col3 = null where (col3) in ::fkc_vals and (:fkc_upd is null or (u_tbl3.col3) not in ((:fkc_upd)))", + "Table": "u_tbl3", + "UpdateExprBvNames": [ + "fkc_upd" + ], + "UpdateExprCols": [ + 1 + ] + }, + { + "InputName": "Parent", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_tbl2 set m = 2, col2 = u_tbl2.col1 + 'bar' where u_tbl2.id = 1", + "Table": "u_tbl2" + } + ] + } + ] + }, + "TablesUsed": [ + "unsharded_fk_allow.u_tbl1", + "unsharded_fk_allow.u_tbl2", + "unsharded_fk_allow.u_tbl3" + ] + } }, { "comment": "update in a table with non-literal value - with cascade fail as the cascade value is not known", @@ -810,8 +890,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select col1, x + 'bar', col1 != x + 'bar' from u_tbl1 where 1 != 1", - "Query": "select col1, x + 'bar', col1 != x + 'bar' from u_tbl1 where id = 1 for update", + "FieldQuery": "select col1, u_tbl1.x + 'bar', col1 != u_tbl1.x + 'bar' from u_tbl1 where 1 != 1", + "Query": "select col1, u_tbl1.x + 'bar', col1 != u_tbl1.x + 'bar' from u_tbl1 where id = 1 for update", "Table": "u_tbl1" }, { @@ -941,7 +1021,7 @@ "Sharded": false }, "TargetTabletType": "PRIMARY", - "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_tbl1 set m = 2, col1 = x + 'bar' where id = 1", + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_tbl1 set m = 2, col1 = u_tbl1.x + 'bar' where id = 1", "Table": "u_tbl1" } ] @@ -1169,7 +1249,85 @@ { "comment": "update with fk on cross-shard with a where condition on non-literal value - disallowed", "query": "update tbl3 set coly = colx + 10 where coly = 10", - "plan": "VT12001: unsupported: update expression with non-literal values with foreign key constraints" + "plan": { + "QueryType": "UPDATE", + "Original": "update tbl3 set coly = colx + 10 where coly = 10", + "Instructions": { + "OperatorType": "FKVerify", + "Inputs": [ + { + "InputName": "VerifyParent-1", + "OperatorType": "Limit", + "Count": "INT64(1)", + "Inputs": [ + { + "OperatorType": "Projection", + "Expressions": [ + "INT64(1) as 1" + ], + "Inputs": [ + { + "OperatorType": "Filter", + "Predicate": "tbl1.t1col1 is null", + "Inputs": [ + { + "OperatorType": "Join", + "Variant": "LeftJoin", + "JoinColumnIndexes": "R:0,R:0", + "JoinVars": { + "tbl3_colx": 0 + }, + "TableName": "tbl3_tbl1", + "Inputs": [ + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "sharded_fk_allow", + "Sharded": true + }, + "FieldQuery": "select tbl3.colx from tbl3 where 1 != 1", + "Query": "select tbl3.colx from tbl3 where tbl3.colx + 10 is not null and tbl3.coly = 10 lock in share mode", + "Table": "tbl3" + }, + { + "OperatorType": "Route", + "Variant": "Scatter", + "Keyspace": { + "Name": "sharded_fk_allow", + "Sharded": true + }, + "FieldQuery": "select tbl1.t1col1 from tbl1 where 1 != 1", + "Query": "select tbl1.t1col1 from tbl1 where tbl1.t1col1 = :tbl3_colx + 10 lock in share mode", + "Table": "tbl1" + } + ] + } + ] + } + ] + } + ] + }, + { + "InputName": "PostVerify", + "OperatorType": "Update", + "Variant": "Scatter", + "Keyspace": { + "Name": "sharded_fk_allow", + "Sharded": true + }, + "TargetTabletType": "PRIMARY", + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ tbl3 set coly = tbl3.colx + 10 where tbl3.coly = 10", + "Table": "tbl3" + } + ] + }, + "TablesUsed": [ + "sharded_fk_allow.tbl1", + "sharded_fk_allow.tbl3" + ] + } }, { "comment": "update with fk on cross-shard with a where condition", @@ -1454,7 +1612,7 @@ "Sharded": false }, "FieldQuery": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = :v1 where 1 != 1", - "Query": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = :v1 where (u_tbl4.col4) in ::fkc_vals and u_tbl3.col3 is null limit 1 lock in share mode", + "Query": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = :v1 where (u_tbl4.col4) in ::fkc_vals and :v1 is not null and u_tbl3.col3 is null limit 1 lock in share mode", "Table": "u_tbl3, u_tbl4" }, { From 44012008f652da9caf86511a4723957cfa0f447a Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 26 Oct 2023 19:10:02 +0530 Subject: [PATCH 15/43] some refactor to club update related offsets Signed-off-by: Harshit Gangal --- go/vt/vtgate/planbuilder/operators/update.go | 77 ++++++++++--------- .../testdata/foreignkey_cases.json | 20 ++--- 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 568243a8c98..8c2a8bcf3b4 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -251,33 +251,26 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, return nil, vterrors.VT13001("ON UPDATE RESTRICT foreign keys should already be filtered") } - nonLiteralUpdate := semantics.HasNonLiteral(updStmt.Exprs, nil, []vindexes.ChildFKInfo{fk}) - // We need to select all the parent columns for the foreign key constraint, to use in the update of the child table. var selectOffsets []int selectOffsets, selectExprs = addColumns(ctx, fk.ParentColumns, selectExprs) - // If we are updating a foreign key column to a non-literal value, then we need to get the updated value and - // whether it is different from the current value or not as well. - var updatedOffsets []int - var compOffsets []int - if nonLiteralUpdate { - // TODO: may only store non-literal update exprs OR store non-literal info along with update expr. - updExprs := ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] - for _, updExpr := range updExprs { - if sqlparser.IsLiteral(updExpr.Expr) { - updatedOffsets = append(updatedOffsets, -1) - compOffsets = append(compOffsets, -1) - continue - } - var updateExprOffset, compExprOffset int - updateExprOffset, compExprOffset, selectExprs = addUpdExprToSelect(ctx, updExpr, selectExprs) - updatedOffsets = append(updatedOffsets, updateExprOffset) - compOffsets = append(compOffsets, compExprOffset) + // If we are updating a foreign key column to a non-literal value then, need information about + // 1. new value is different from the old value + // 2. the new value itself + var updFkColOffsets [][2]int + // TODO: may only store non-literal update exprs OR store non-literal info along with update expr. + updExprs := ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] + for _, updExpr := range updExprs { + // TODO: not sure why we append for literals. + offset := [2]int{-1, -1} + if !sqlparser.IsLiteral(updExpr.Expr) { + offset, selectExprs = addUpdExprToSelect(ctx, updExpr, selectExprs) } + updFkColOffsets = append(updFkColOffsets, offset) } - fkChild, err := createFkChildForUpdate(ctx, fk, updStmt, selectOffsets, updatedOffsets, compOffsets, updatedTable) + fkChild, err := createFkChildForUpdate(ctx, fk, updStmt, selectOffsets, updFkColOffsets, updatedTable) if err != nil { return nil, err } @@ -318,12 +311,26 @@ func addColumns(ctx *plancontext.PlanningContext, columns sqlparser.Columns, exp return offsets, selectExprs } -func addUpdExprToSelect(ctx *plancontext.PlanningContext, updExpr *sqlparser.UpdateExpr, exprs []sqlparser.SelectExpr) (int, int, []sqlparser.SelectExpr) { - var updateExprOffset, compExprOffset int - updateExprOffset, exprs = addExprToSelect(ctx, updExpr.Expr, exprs) +func addUpdExprToSelect(ctx *plancontext.PlanningContext, updExpr *sqlparser.UpdateExpr, exprs []sqlparser.SelectExpr) ([2]int, []sqlparser.SelectExpr) { compExpr := sqlparser.NewComparisonExpr(sqlparser.NotEqualOp, updExpr.Name, updExpr.Expr, nil) - compExprOffset, exprs = addExprToSelect(ctx, compExpr, exprs) - return updateExprOffset, compExprOffset, exprs + offsets := [2]int{-1, -1} + for idx, selectExpr := range exprs { + if ctx.SemTable.EqualsExpr(selectExpr.(*sqlparser.AliasedExpr).Expr, compExpr) { + offsets[0] = idx + } + if ctx.SemTable.EqualsExpr(selectExpr.(*sqlparser.AliasedExpr).Expr, updExpr.Expr) { + offsets[1] = idx + } + } + if offsets[0] == -1 { + offsets[0] = len(exprs) + exprs = append(exprs, aeWrap(compExpr)) + } + if offsets[1] == -1 { + offsets[1] = len(exprs) + exprs = append(exprs, aeWrap(updExpr.Expr)) + } + return offsets, exprs } func addExprToSelect(ctx *plancontext.PlanningContext, expr sqlparser.Expr, exprs []sqlparser.SelectExpr) (int, []sqlparser.SelectExpr) { @@ -337,7 +344,7 @@ func addExprToSelect(ctx *plancontext.PlanningContext, expr sqlparser.Expr, expr } // createFkChildForUpdate creates the update query operator for the child table based on the foreign key constraints. -func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, selectOffsets, updatedOffsets, compOffsets []int, updatedTable *vindexes.Table) (*FkChild, error) { +func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, selectOffsets []int, updFkColOffsets [][2]int, updatedTable *vindexes.Table) (*FkChild, error) { // Create a ValTuple of child column names var valTuple sqlparser.ValTuple for _, column := range fk.ChildColumns { @@ -351,9 +358,9 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF var childWhereExpr sqlparser.Expr = compExpr var updateExprBvNames []string - if len(updatedOffsets) > 0 { - for _, updateOffset := range updatedOffsets { - if updateOffset == -1 { + if len(updFkColOffsets) > 0 { + for _, offset := range updFkColOffsets { + if offset[1] == -1 { updateExprBvNames = append(updateExprBvNames, "") continue } @@ -376,7 +383,7 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF return nil, err } - updatedOffsets, compOffsets, updateExprBvNames = compressUpdateOffsets(updatedOffsets, compOffsets, updateExprBvNames) + updatedOffsets, compOffsets, updateExprBvNames := compressUpdateOffsets(updFkColOffsets, updateExprBvNames) return &FkChild{ BVName: bvName, @@ -388,15 +395,15 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF }, nil } -func compressUpdateOffsets(updatedOffsets []int, compOffsets []int, updateExprBvNames []string) ([]int, []int, []string) { +func compressUpdateOffsets(offsets [][2]int, updateExprBvNames []string) ([]int, []int, []string) { var newUpdatedOffsets, newCompOffsets []int var newUpdateExprBvNames []string - for idx, updateOffset := range updatedOffsets { - if updateOffset == -1 { + for idx, offset := range offsets { + if offset[1] == -1 { continue } - newUpdatedOffsets = append(newUpdatedOffsets, updateOffset) - newCompOffsets = append(newCompOffsets, compOffsets[idx]) + newUpdatedOffsets = append(newUpdatedOffsets, offset[1]) + newCompOffsets = append(newCompOffsets, offset[0]) newUpdateExprBvNames = append(newUpdateExprBvNames, updateExprBvNames[idx]) } return newUpdatedOffsets, newCompOffsets, newUpdateExprBvNames diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index a4cc831ad20..5dc095022bf 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -821,8 +821,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select col2, u_tbl2.col1 + 'bar', col2 != u_tbl2.col1 + 'bar' from u_tbl2 where 1 != 1", - "Query": "select col2, u_tbl2.col1 + 'bar', col2 != u_tbl2.col1 + 'bar' from u_tbl2 where u_tbl2.id = 1 for update", + "FieldQuery": "select col2, col2 != u_tbl2.col1 + 'bar', u_tbl2.col1 + 'bar' from u_tbl2 where 1 != 1", + "Query": "select col2, col2 != u_tbl2.col1 + 'bar', u_tbl2.col1 + 'bar' from u_tbl2 where u_tbl2.id = 1 for update", "Table": "u_tbl2" }, { @@ -839,7 +839,7 @@ 0 ], "CompExprCols": [ - 2 + 1 ], "Query": "update u_tbl3 set col3 = null where (col3) in ::fkc_vals and (:fkc_upd is null or (u_tbl3.col3) not in ((:fkc_upd)))", "Table": "u_tbl3", @@ -847,7 +847,7 @@ "fkc_upd" ], "UpdateExprCols": [ - 1 + 2 ] }, { @@ -890,8 +890,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select col1, u_tbl1.x + 'bar', col1 != u_tbl1.x + 'bar' from u_tbl1 where 1 != 1", - "Query": "select col1, u_tbl1.x + 'bar', col1 != u_tbl1.x + 'bar' from u_tbl1 where id = 1 for update", + "FieldQuery": "select col1, col1 != u_tbl1.x + 'bar', u_tbl1.x + 'bar' from u_tbl1 where 1 != 1", + "Query": "select col1, col1 != u_tbl1.x + 'bar', u_tbl1.x + 'bar' from u_tbl1 where id = 1 for update", "Table": "u_tbl1" }, { @@ -902,13 +902,13 @@ 0 ], "CompExprCols": [ - 2 + 1 ], "UpdateExprBvNames": [ "fkc_upd" ], "UpdateExprCols": [ - 1 + 2 ], "Inputs": [ { @@ -961,13 +961,13 @@ 0 ], "CompExprCols": [ - 2 + 1 ], "UpdateExprBvNames": [ "fkc_upd1" ], "UpdateExprCols": [ - 1 + 2 ], "Inputs": [ { From adfd62f439a98641cfbd522c27494a413057b098 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 26 Oct 2023 23:16:06 +0530 Subject: [PATCH 16/43] reject queries with fk column update dependent column also getting updated Signed-off-by: Harshit Gangal --- go/vt/vtgate/planbuilder/operators/update.go | 19 +-- .../testdata/foreignkey_cases.json | 113 +++++++++++++++++- go/vt/vtgate/semantics/analyzer.go | 40 ++++++- go/vt/vtgate/semantics/semantic_state.go | 7 +- 4 files changed, 159 insertions(+), 20 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 8c2a8bcf3b4..3f366aad555 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -260,8 +260,11 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, // 2. the new value itself var updFkColOffsets [][2]int // TODO: may only store non-literal update exprs OR store non-literal info along with update expr. - updExprs := ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] - for _, updExpr := range updExprs { + ue := ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] + if ue.DependencyUpdated { + return nil, vterrors.VT12001("same column referenced in foreign key column update is also updated") + } + for _, updExpr := range ue.Exprs { // TODO: not sure why we append for literals. offset := [2]int{-1, -1} if !sqlparser.IsLiteral(updExpr.Expr) { @@ -373,9 +376,9 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF var err error switch fk.OnUpdate { case sqlparser.Cascade: - childOp, err = buildChildUpdOpForCascade(ctx, fk, updStmt, childWhereExpr, updateExprBvNames, updatedTable) + childOp, err = buildChildUpdOpForCascade(ctx, fk, childWhereExpr, updateExprBvNames, updatedTable) case sqlparser.SetNull: - childOp, err = buildChildUpdOpForSetNull(ctx, fk, updStmt, childWhereExpr, updateExprBvNames, updatedTable) + childOp, err = buildChildUpdOpForSetNull(ctx, fk, childWhereExpr, updateExprBvNames, updatedTable) case sqlparser.SetDefault: return nil, vterrors.VT09016() } @@ -413,11 +416,11 @@ func compressUpdateOffsets(offsets [][2]int, updateExprBvNames []string) ([]int, // The query looks like this - // // `UPDATE SET WHERE IN ()` -func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, childWhereExpr sqlparser.Expr, updatedExprBvNames []string, updatedTable *vindexes.Table) (ops.Operator, error) { +func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, childWhereExpr sqlparser.Expr, updatedExprBvNames []string, updatedTable *vindexes.Table) (ops.Operator, error) { // The update expressions are the same as the update expressions in the parent update query // with the column names replaced with the child column names. var childUpdateExprs sqlparser.UpdateExprs - for idx, updateExpr := range ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] { + for idx, updateExpr := range ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)].Exprs { colIdx := fk.ParentColumns.FindColumn(updateExpr.Name.Name) if colIdx == -1 { continue @@ -458,7 +461,7 @@ func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.Chi // `UPDATE SET // WHERE IN () // [AND ({ IS NULL OR}... NOT IN ())]` -func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, childWhereExpr sqlparser.Expr, updateExprBvNames []string, updatedTable *vindexes.Table) (ops.Operator, error) { +func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, childWhereExpr sqlparser.Expr, updateExprBvNames []string, updatedTable *vindexes.Table) (ops.Operator, error) { // For the SET NULL type constraint, we need to set all the child columns to NULL. var childUpdateExprs sqlparser.UpdateExprs for _, column := range fk.ChildColumns { @@ -477,7 +480,7 @@ func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.Chi // For example, if we are setting `update parent cola = :v1 and colb = :v2`, then on the child, the where condition would look something like this - // `:v1 IS NULL OR :v2 IS NULL OR (child_cola, child_colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL). - compExpr := nullSafeNotInComparison(ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)], fk, updatedTable.GetTableName(), updateExprBvNames) + compExpr := nullSafeNotInComparison(ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)].Exprs, fk, updatedTable.GetTableName(), updateExprBvNames) if compExpr != nil { childWhereExpr = &sqlparser.AndExpr{ Left: childWhereExpr, diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index 5dc095022bf..47597c9377e 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -789,7 +789,7 @@ "plan": "VT12001: unsupported: update with limit with foreign key constraints" }, { - "comment": "update in a table with non-literal value - set null fail due to child update where condition", + "comment": "update in a table with non-literal value - set null", "query": "update u_tbl2 set m = 2, col2 = col1 + 'bar' where id = 1", "plan": { "QueryType": "UPDATE", @@ -874,7 +874,7 @@ } }, { - "comment": "update in a table with non-literal value - with cascade fail as the cascade value is not known", + "comment": "update in a table with non-literal value - with cascade", "query": "update u_tbl1 set m = 2, col1 = x + 'bar' where id = 1", "plan": { "QueryType": "UPDATE", @@ -1036,7 +1036,7 @@ } }, { - "comment": "update in a table with set null, non-literal value on non-foreign key column - allowed", + "comment": "update in a table with set null, non-literal value on non-foreign key column", "query": "update u_tbl2 set m = col1 + 'bar', col2 = 2 where id = 1", "plan": { "QueryType": "UPDATE", @@ -1093,7 +1093,7 @@ } }, { - "comment": "update in a table with cascade, non-literal value on non-foreign key column - allowed", + "comment": "update in a table with cascade, non-literal value on non-foreign key column", "query": "update u_tbl1 set m = x + 'bar', col1 = 2 where id = 1", "plan": { "QueryType": "UPDATE", @@ -1247,7 +1247,7 @@ "plan": "VT12001: unsupported: foreign keys management at vitess with limit" }, { - "comment": "update with fk on cross-shard with a where condition on non-literal value - disallowed", + "comment": "update with fk on cross-shard with a update condition on non-literal value", "query": "update tbl3 set coly = colx + 10 where coly = 10", "plan": { "QueryType": "UPDATE", @@ -1968,5 +1968,108 @@ "sharded_fk_allow.tbl5" ] } + }, + { + "comment": "foreign key column updated by using a column which is also getting updated", + "query": "update u_tbl1 set foo = 100, col1 = baz + 1 + foo where bar = 42", + "plan": "VT12001: unsupported: same column referenced in foreign key column update is also updated" + }, + { + "comment": "foreign key column updated by using a column which is also getting updated - self reference column is allowed", + "query": "update u_tbl7 set foo = 100, col7 = baz + 1 + col7 where bar = 42", + "plan": { + "QueryType": "UPDATE", + "Original": "update u_tbl7 set foo = 100, col7 = baz + 1 + col7 where bar = 42", + "Instructions": { + "OperatorType": "FkCascade", + "Inputs": [ + { + "InputName": "Selection", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select col7, col7 != baz + 1 + col7, baz + 1 + col7 from u_tbl7 where 1 != 1", + "Query": "select col7, col7 != baz + 1 + col7, baz + 1 + col7 from u_tbl7 where bar = 42 for update", + "Table": "u_tbl7" + }, + { + "InputName": "CascadeChild-1", + "OperatorType": "FKVerify", + "BvName": "fkc_vals", + "Cols": [ + 0 + ], + "CompExprCols": [ + 1 + ], + "UpdateExprBvNames": [ + "fkc_upd" + ], + "UpdateExprCols": [ + 2 + ], + "Inputs": [ + { + "InputName": "VerifyParent-1", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = :fkc_upd where 1 != 1", + "Query": "select 1 from u_tbl4 left join u_tbl3 on u_tbl3.col3 = :fkc_upd where (u_tbl4.col4) in ::fkc_vals and :fkc_upd is not null and u_tbl3.col3 is null limit 1 lock in share mode", + "Table": "u_tbl3, u_tbl4" + }, + { + "InputName": "VerifyChild-2", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select 1 from u_tbl4, u_tbl9 where 1 != 1", + "Query": "select 1 from u_tbl4, u_tbl9 where (u_tbl4.col4) in ::fkc_vals and (:fkc_upd is null or (u_tbl9.col9) not in ((:fkc_upd))) and u_tbl4.col4 = u_tbl9.col9 limit 1 lock in share mode", + "Table": "u_tbl4, u_tbl9" + }, + { + "InputName": "PostVerify", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_tbl4 set col4 = :fkc_upd where (u_tbl4.col4) in ::fkc_vals", + "Table": "u_tbl4" + } + ] + }, + { + "InputName": "Parent", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_tbl7 set foo = 100, col7 = baz + 1 + col7 where bar = 42", + "Table": "u_tbl7" + } + ] + }, + "TablesUsed": [ + "unsharded_fk_allow.u_tbl3", + "unsharded_fk_allow.u_tbl4", + "unsharded_fk_allow.u_tbl7", + "unsharded_fk_allow.u_tbl9" + ] + } } ] diff --git a/go/vt/vtgate/semantics/analyzer.go b/go/vt/vtgate/semantics/analyzer.go index 1a66590e0ae..fff2f015166 100644 --- a/go/vt/vtgate/semantics/analyzer.go +++ b/go/vt/vtgate/semantics/analyzer.go @@ -319,7 +319,7 @@ func (a *analyzer) noteQuerySignature(node sqlparser.SQLNode) { } // getInvolvedForeignKeys gets the foreign keys that might require taking care off when executing the given statement. -func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs, bool, error) { +func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]*UpdateExpression, bool, error) { // There are only the DML statements that require any foreign keys handling. switch stmt := statement.(type) { case *sqlparser.Delete: @@ -355,6 +355,7 @@ func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[Ta return nil, nil, nil, false, err } childFks, parentFks, childFkToUpdExprs := a.filterForeignKeysUsingUpdateExpressions(allChildFks, allParentFks, stmt.Exprs) + // TODO: add comment for turning off the foreign_key_checks fkChecksOff := false if HasNonLiteral(stmt.Exprs, collectParentFksFromMap(parentFks), collectChildFksFromMap(childFks)) { fkChecksOff = true @@ -401,7 +402,7 @@ func HasNonLiteral(updExprs sqlparser.UpdateExprs, parentFks []vindexes.ParentFK } // filterForeignKeysUsingUpdateExpressions filters the child and parent foreign key constraints that don't require any validations/cascades given the updated expressions. -func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[TableSet][]vindexes.ChildFKInfo, allParentFks map[TableSet][]vindexes.ParentFKInfo, updExprs sqlparser.UpdateExprs) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs) { +func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[TableSet][]vindexes.ChildFKInfo, allParentFks map[TableSet][]vindexes.ParentFKInfo, updExprs sqlparser.UpdateExprs) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]*UpdateExpression) { if len(allChildFks) == 0 && len(allParentFks) == 0 { return nil, nil, nil } @@ -419,7 +420,7 @@ func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[Table updExprToTableSet := make(map[*sqlparser.ColName]TableSet) // childFKToUpdExprs stores child foreign key to update expressions mapping. - childFKToUpdExprs := map[string]sqlparser.UpdateExprs{} + childFKToUpdExprs := map[string]*UpdateExpression{} // Go over all the update expressions for _, updateExpr := range updExprs { @@ -438,7 +439,14 @@ func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[Table if childFk.ParentColumns.FindColumn(updateExpr.Name.Name) >= 0 { cFksRequired[deps][idx] = true tbl, _ := a.tables.tableInfoFor(deps) - childFKToUpdExprs[childFk.String(tbl.GetVindexTable())] = append(childFKToUpdExprs[childFk.String(tbl.GetVindexTable())], updateExpr) + ue, exists := childFKToUpdExprs[childFk.String(tbl.GetVindexTable())] + if !exists { + ue = &UpdateExpression{} + childFKToUpdExprs[childFk.String(tbl.GetVindexTable())] = ue + } + ue.Exprs = append(ue.Exprs, updateExpr) + ue.DependencyUpdated = ue.DependencyUpdated || isDependentColumnUpdated(updateExpr, updExprs) + } } // If we are setting a column to NULL, then we don't need to verify the existance of an @@ -480,7 +488,6 @@ func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[Table } } pFksNeedsHandling[ts] = pFKNeeded - } for ts, childFks := range allChildFks { var cFKNeeded []vindexes.ChildFKInfo @@ -490,11 +497,32 @@ func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[Table } } cFksNeedsHandling[ts] = cFKNeeded - } return cFksNeedsHandling, pFksNeedsHandling, childFKToUpdExprs } +func isDependentColumnUpdated(ue *sqlparser.UpdateExpr, updExprs sqlparser.UpdateExprs) bool { + dependencyUpdated := false + _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { + col, ok := node.(*sqlparser.ColName) + if !ok { + return true, nil + } + // self reference column dependency is not considered a dependent column being updated. + if ue.Name.Equal(col) { + return true, nil + } + for _, updExpr := range updExprs { + if updExpr.Name.Equal(col) { + dependencyUpdated = true + return false, nil + } + } + return false, nil + }, ue.Expr) + return dependencyUpdated +} + // getAllManagedForeignKeys gets all the foreign keys for the query we are analyzing that Vitess is reposible for managing. func (a *analyzer) getAllManagedForeignKeys() (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, error) { allChildFKs := make(map[TableSet][]vindexes.ChildFKInfo) diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index f23c7f88d7e..78027430e43 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -132,7 +132,7 @@ type ( // The map is keyed by the tableset of the table that each of the foreign key belongs to. childForeignKeysInvolved map[TableSet][]vindexes.ChildFKInfo parentForeignKeysInvolved map[TableSet][]vindexes.ParentFKInfo - ChildFkToUpdExprs map[string]sqlparser.UpdateExprs + ChildFkToUpdExprs map[string]*UpdateExpression FKChecksOff bool } @@ -141,6 +141,11 @@ type ( ColumnName string } + UpdateExpression struct { + Exprs sqlparser.UpdateExprs + DependencyUpdated bool + } + // SchemaInformation is used tp provide table information from Vschema. SchemaInformation interface { FindTableOrVindex(tablename sqlparser.TableName) (*vindexes.Table, vindexes.Vindex, string, topodatapb.TabletType, key.Destination, error) From 2a04ecf19b6c8c211fd230c6fc42260bfc6803c2 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Oct 2023 12:23:53 +0530 Subject: [PATCH 17/43] fuzz: fix fuzzer to only produce queries that work with Vitess and Mysql for single threaded mode Signed-off-by: Manan Gupta --- .../vtgate/foreignkey/fk_fuzz_test.go | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go index bf3a1ffc68a..80defdcf0a6 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go @@ -146,16 +146,26 @@ func (fz *fuzzer) generateUpdateDMLQuery() string { idValue := 1 + rand.Intn(fz.maxValForId) tableName := fkTables[tableId] if tableName == "fk_t20" { - colValue := fz.generateExpression(rand.Intn(4)+1, "col", "col2") - col2Value := fz.generateExpression(rand.Intn(4)+1, "col", "col2") + colValue := convertIntValueToString(rand.Intn(1 + fz.maxValForCol)) + col2Value := convertIntValueToString(rand.Intn(1 + fz.maxValForCol)) return fmt.Sprintf("update %v set col = %v, col2 = %v where id = %v", tableName, colValue, col2Value, idValue) } else if isMultiColFkTable(tableName) { - colaValue := fz.generateExpression(rand.Intn(4)+1, "cola", "colb") - colbValue := fz.generateExpression(rand.Intn(4)+1, "cola", "colb") - return fmt.Sprintf("update %v set cola = %v, colb = %v where id = %v", tableName, colaValue, colbValue, idValue) + if rand.Intn(2) == 0 { + colaValue := convertIntValueToString(rand.Intn(1 + fz.maxValForCol)) + colbValue := convertIntValueToString(rand.Intn(1 + fz.maxValForCol)) + if fz.concurrency > 1 { + colaValue = fz.generateExpression(rand.Intn(4)+1, "cola", "colb", "id") + colbValue = fz.generateExpression(rand.Intn(4)+1, "cola", "colb", "id") + } + return fmt.Sprintf("update %v set cola = %v, colb = %v where id = %v", tableName, colaValue, colbValue, idValue) + } else { + colValue := fz.generateExpression(rand.Intn(4)+1, "cola", "colb", "id") + colToUpdate := []string{"cola", "colb"}[rand.Intn(2)] + return fmt.Sprintf("update %v set %v = %v where id = %v", tableName, colToUpdate, colValue, idValue) + } } else { - colValue := rand.Intn(1 + fz.maxValForCol) - return fmt.Sprintf("update %v set col = %v where id = %v", tableName, convertIntValueToString(colValue), idValue) + colValue := fz.generateExpression(rand.Intn(4)+1, "col", "id") + return fmt.Sprintf("update %v set col = %v where id = %v", tableName, colValue, idValue) } } From 474a337413355c5d3787946418628c0db27e992b Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Oct 2023 13:37:33 +0530 Subject: [PATCH 18/43] feat: fix update planning to produce correct not in tuple comparison Signed-off-by: Manan Gupta --- go/test/endtoend/vtgate/foreignkey/fk_test.go | 22 ++++ go/vt/vtgate/planbuilder/operators/update.go | 17 ++- .../testdata/foreignkey_cases.json | 103 ++++++++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/go/test/endtoend/vtgate/foreignkey/fk_test.go b/go/test/endtoend/vtgate/foreignkey/fk_test.go index 1fff5d279d8..bc358a27dca 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_test.go @@ -834,6 +834,28 @@ func TestFkQueries(t *testing.T) { "insert into fk_t13 (id, col) values (1,1),(2,2)", "update fk_t11 set col = id + 3", }, + }, { + name: "Single column update in a multi-col table - success", + queries: []string{ + "insert into fk_multicol_t1 (id, cola, colb) values (1, 1, 1), (2, 2, 2)", + "insert into fk_multicol_t2 (id, cola, colb) values (1, 1, 1)", + "update fk_multicol_t1 set colb = 4 + (colb) where id = 2", + }, + }, { + name: "Single column update in a multi-col table - restrict failure", + queries: []string{ + "insert into fk_multicol_t1 (id, cola, colb) values (1, 1, 1), (2, 2, 2)", + "insert into fk_multicol_t2 (id, cola, colb) values (1, 1, 1)", + "update fk_multicol_t1 set colb = 4 + (colb) where id = 1", + }, + }, { + name: "Single column update in multi-col table - cascade and set null", + queries: []string{ + "insert into fk_multicol_t15 (id, cola, colb) values (1, 1, 1), (2, 2, 2)", + "insert into fk_multicol_t16 (id, cola, colb) values (1, 1, 1), (2, 2, 2)", + "insert into fk_multicol_t17 (id, cola, colb) values (1, 1, 1), (2, 2, 2)", + "update fk_multicol_t15 set colb = 4 + (colb) where id = 1", + }, }, } diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 3f366aad555..737fd95ec2b 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -696,6 +696,8 @@ func createFkVerifyOpForChildFKForUpdate(ctx *plancontext.PlanningContext, updSt // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL) // This expression is used in cascading SET NULLs and in verifying whether an update should be restricted. func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.ChildFKInfo, parentTbl sqlparser.TableName, updatedExprBvNames []string) sqlparser.Expr { + //var updateValues sqlparser.ValTuple = make([]sqlparser.Expr, len(cFk.ChildColumns)) + var valTuple sqlparser.ValTuple var updateValues sqlparser.ValTuple for idx, updateExpr := range updateExprs { colIdx := cFk.ParentColumns.FindColumn(updateExpr.Name.Name) @@ -707,14 +709,21 @@ func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.Chi if len(updatedExprBvNames) > 0 && updatedExprBvNames[idx] != "" { childUpdateExpr = sqlparser.NewArgument(updatedExprBvNames[idx]) } + //updateValues[colIdx] = childUpdateExpr updateValues = append(updateValues, childUpdateExpr) + valTuple = append(valTuple, sqlparser.NewColNameWithQualifier(cFk.ChildColumns[colIdx].String(), cFk.Table.GetTableName())) } } + //for idx, value := range updateValues { + // if value == nil { + // updateValues[idx] = sqlparser.NewColNameWithQualifier(cFk.ChildColumns[idx].String(), cFk.Table.GetTableName()) + // } + //} // Create a ValTuple of child column names - var valTuple sqlparser.ValTuple - for _, column := range cFk.ChildColumns { - valTuple = append(valTuple, sqlparser.NewColNameWithQualifier(column.String(), cFk.Table.GetTableName())) - } + //var valTuple sqlparser.ValTuple + //for _, column := range cFk.ChildColumns { + // valTuple = append(valTuple, sqlparser.NewColNameWithQualifier(column.String(), cFk.Table.GetTableName())) + //} var finalExpr sqlparser.Expr = sqlparser.NewComparisonExpr(sqlparser.NotInOp, valTuple, sqlparser.ValTuple{updateValues}, nil) for _, value := range updateValues { finalExpr = &sqlparser.OrExpr{ diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index 47597c9377e..1260ad8d76d 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -2071,5 +2071,108 @@ "unsharded_fk_allow.u_tbl9" ] } + }, + { + "comment": "Single column updated in a multi-col table", + "query": "update u_multicol_tbl1 set cola = cola + 3 where id = 3", + "plan": { + "QueryType": "UPDATE", + "Original": "update u_multicol_tbl1 set cola = cola + 3 where id = 3", + "Instructions": { + "OperatorType": "FkCascade", + "Inputs": [ + { + "InputName": "Selection", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select cola, colb, cola != u_multicol_tbl1.cola + 3, u_multicol_tbl1.cola + 3 from u_multicol_tbl1 where 1 != 1", + "Query": "select cola, colb, cola != u_multicol_tbl1.cola + 3, u_multicol_tbl1.cola + 3 from u_multicol_tbl1 where id = 3 for update", + "Table": "u_multicol_tbl1" + }, + { + "InputName": "CascadeChild-1", + "OperatorType": "FkCascade", + "BvName": "fkc_vals", + "Cols": [ + 0, + 1 + ], + "CompExprCols": [ + 2 + ], + "UpdateExprBvNames": [ + "fkc_upd" + ], + "UpdateExprCols": [ + 3 + ], + "Inputs": [ + { + "InputName": "Selection", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select cola, colb from u_multicol_tbl2 where 1 != 1", + "Query": "select cola, colb from u_multicol_tbl2 where (cola, colb) in ::fkc_vals and (:fkc_upd is null or (u_multicol_tbl2.cola) not in ((:fkc_upd))) for update", + "Table": "u_multicol_tbl2" + }, + { + "InputName": "CascadeChild-1", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "BvName": "fkc_vals1", + "Cols": [ + 0, + 1 + ], + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_multicol_tbl3 set cola = null, colb = null where (cola, colb) in ::fkc_vals1", + "Table": "u_multicol_tbl3" + }, + { + "InputName": "Parent", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update u_multicol_tbl2 set cola = null, colb = null where (cola, colb) in ::fkc_vals and (:fkc_upd is null or (u_multicol_tbl2.cola) not in ((:fkc_upd)))", + "Table": "u_multicol_tbl2" + } + ] + }, + { + "InputName": "Parent", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_multicol_tbl1 set cola = u_multicol_tbl1.cola + 3 where id = 3", + "Table": "u_multicol_tbl1" + } + ] + }, + "TablesUsed": [ + "unsharded_fk_allow.u_multicol_tbl1", + "unsharded_fk_allow.u_multicol_tbl2", + "unsharded_fk_allow.u_multicol_tbl3" + ] + } } ] From 1ec7b00d194d450da73ff5f4295300686f3cae8f Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 27 Oct 2023 13:44:42 +0530 Subject: [PATCH 19/43] feat: fix fk_cascade engine to handle null comparisons correctly Signed-off-by: Manan Gupta --- go/test/endtoend/vtgate/foreignkey/fk_test.go | 16 ++++++++++++++++ go/vt/vtgate/engine/fk_cascade.go | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/vtgate/foreignkey/fk_test.go b/go/test/endtoend/vtgate/foreignkey/fk_test.go index bc358a27dca..db2894fe75b 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_test.go @@ -856,6 +856,22 @@ func TestFkQueries(t *testing.T) { "insert into fk_multicol_t17 (id, cola, colb) values (1, 1, 1), (2, 2, 2)", "update fk_multicol_t15 set colb = 4 + (colb) where id = 1", }, + }, { + name: "Non literal update that evaluates to NULL - restricted", + queries: []string{ + "insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t13 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "update fk_t10 set col = id + null where id = 1", + }, + }, { + name: "Non literal update that evaluates to NULL - success", + queries: []string{ + "insert into fk_t10 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t11 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", + "update fk_t10 set col = id + null where id = 1", + }, }, } diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index e7eeebfdba7..7d87d808f27 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -133,7 +133,8 @@ func (fkc *FkCascade) executeNonLiteralUpdateFkChild(ctx context.Context, vcurso skipRow := true for _, colIdx := range child.CompExprCols { if row[colIdx].IsNull() { - continue + skipRow = false + break } hasChanged, err := row[colIdx].ToBool() if err != nil { From 5f9ad38f3b5cac8f6befd34b20f0c0cac83fe7c6 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 30 Oct 2023 10:32:01 +0530 Subject: [PATCH 20/43] feat: refactor and fix update expressions dependency logic Signed-off-by: Manan Gupta --- go/vt/vtgate/planbuilder/operators/update.go | 17 ++++-- .../testdata/foreignkey_cases.json | 5 ++ go/vt/vtgate/semantics/analyzer.go | 39 ++---------- go/vt/vtgate/semantics/semantic_state.go | 60 ++++++++++++++++++- 4 files changed, 81 insertions(+), 40 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 737fd95ec2b..1f06261b570 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -125,6 +125,14 @@ func createOperatorFromUpdate(ctx *plancontext.PlanningContext, updStmt *sqlpars return nil, vterrors.VT12001("update with limit with foreign key constraints") } + depUpd, err := ctx.SemTable.IsFkDependentColumnUpdated(updStmt.Exprs) + if err != nil { + return nil, err + } + if depUpd { + return nil, vterrors.VT12001("same column referenced in foreign key column update is also updated") + } + return buildFkOperator(ctx, updOp, updClone, parentFks, childFks, vindexTable) } @@ -261,10 +269,7 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, var updFkColOffsets [][2]int // TODO: may only store non-literal update exprs OR store non-literal info along with update expr. ue := ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] - if ue.DependencyUpdated { - return nil, vterrors.VT12001("same column referenced in foreign key column update is also updated") - } - for _, updExpr := range ue.Exprs { + for _, updExpr := range ue { // TODO: not sure why we append for literals. offset := [2]int{-1, -1} if !sqlparser.IsLiteral(updExpr.Expr) { @@ -420,7 +425,7 @@ func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.Chi // The update expressions are the same as the update expressions in the parent update query // with the column names replaced with the child column names. var childUpdateExprs sqlparser.UpdateExprs - for idx, updateExpr := range ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)].Exprs { + for idx, updateExpr := range ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] { colIdx := fk.ParentColumns.FindColumn(updateExpr.Name.Name) if colIdx == -1 { continue @@ -480,7 +485,7 @@ func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.Chi // For example, if we are setting `update parent cola = :v1 and colb = :v2`, then on the child, the where condition would look something like this - // `:v1 IS NULL OR :v2 IS NULL OR (child_cola, child_colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL). - compExpr := nullSafeNotInComparison(ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)].Exprs, fk, updatedTable.GetTableName(), updateExprBvNames) + compExpr := nullSafeNotInComparison(ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)], fk, updatedTable.GetTableName(), updateExprBvNames) if compExpr != nil { childWhereExpr = &sqlparser.AndExpr{ Left: childWhereExpr, diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index 1260ad8d76d..7b6e1fe1ff4 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -2174,5 +2174,10 @@ "unsharded_fk_allow.u_multicol_tbl3" ] } + }, + { + "comment": "updating multiple columns of a fk constraint such that one uses the other", + "query": "update u_multicol_tbl3 set cola = id, colb = 5 * (cola + (1 - (cola))) where id = 2", + "plan": "VT12001: unsupported: same column referenced in foreign key column update is also updated" } ] diff --git a/go/vt/vtgate/semantics/analyzer.go b/go/vt/vtgate/semantics/analyzer.go index fff2f015166..c5a22447f63 100644 --- a/go/vt/vtgate/semantics/analyzer.go +++ b/go/vt/vtgate/semantics/analyzer.go @@ -319,7 +319,7 @@ func (a *analyzer) noteQuerySignature(node sqlparser.SQLNode) { } // getInvolvedForeignKeys gets the foreign keys that might require taking care off when executing the given statement. -func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]*UpdateExpression, bool, error) { +func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs, bool, error) { // There are only the DML statements that require any foreign keys handling. switch stmt := statement.(type) { case *sqlparser.Delete: @@ -402,7 +402,7 @@ func HasNonLiteral(updExprs sqlparser.UpdateExprs, parentFks []vindexes.ParentFK } // filterForeignKeysUsingUpdateExpressions filters the child and parent foreign key constraints that don't require any validations/cascades given the updated expressions. -func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[TableSet][]vindexes.ChildFKInfo, allParentFks map[TableSet][]vindexes.ParentFKInfo, updExprs sqlparser.UpdateExprs) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]*UpdateExpression) { +func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[TableSet][]vindexes.ChildFKInfo, allParentFks map[TableSet][]vindexes.ParentFKInfo, updExprs sqlparser.UpdateExprs) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs) { if len(allChildFks) == 0 && len(allParentFks) == 0 { return nil, nil, nil } @@ -420,7 +420,7 @@ func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[Table updExprToTableSet := make(map[*sqlparser.ColName]TableSet) // childFKToUpdExprs stores child foreign key to update expressions mapping. - childFKToUpdExprs := map[string]*UpdateExpression{} + childFKToUpdExprs := map[string]sqlparser.UpdateExprs{} // Go over all the update expressions for _, updateExpr := range updExprs { @@ -439,14 +439,9 @@ func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[Table if childFk.ParentColumns.FindColumn(updateExpr.Name.Name) >= 0 { cFksRequired[deps][idx] = true tbl, _ := a.tables.tableInfoFor(deps) - ue, exists := childFKToUpdExprs[childFk.String(tbl.GetVindexTable())] - if !exists { - ue = &UpdateExpression{} - childFKToUpdExprs[childFk.String(tbl.GetVindexTable())] = ue - } - ue.Exprs = append(ue.Exprs, updateExpr) - ue.DependencyUpdated = ue.DependencyUpdated || isDependentColumnUpdated(updateExpr, updExprs) - + ue := childFKToUpdExprs[childFk.String(tbl.GetVindexTable())] + ue = append(ue, updateExpr) + childFKToUpdExprs[childFk.String(tbl.GetVindexTable())] = ue } } // If we are setting a column to NULL, then we don't need to verify the existance of an @@ -501,28 +496,6 @@ func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[Table return cFksNeedsHandling, pFksNeedsHandling, childFKToUpdExprs } -func isDependentColumnUpdated(ue *sqlparser.UpdateExpr, updExprs sqlparser.UpdateExprs) bool { - dependencyUpdated := false - _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { - col, ok := node.(*sqlparser.ColName) - if !ok { - return true, nil - } - // self reference column dependency is not considered a dependent column being updated. - if ue.Name.Equal(col) { - return true, nil - } - for _, updExpr := range updExprs { - if updExpr.Name.Equal(col) { - dependencyUpdated = true - return false, nil - } - } - return false, nil - }, ue.Expr) - return dependencyUpdated -} - // getAllManagedForeignKeys gets all the foreign keys for the query we are analyzing that Vitess is reposible for managing. func (a *analyzer) getAllManagedForeignKeys() (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, error) { allChildFKs := make(map[TableSet][]vindexes.ChildFKInfo) diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 78027430e43..9a22be80423 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -132,7 +132,7 @@ type ( // The map is keyed by the tableset of the table that each of the foreign key belongs to. childForeignKeysInvolved map[TableSet][]vindexes.ChildFKInfo parentForeignKeysInvolved map[TableSet][]vindexes.ParentFKInfo - ChildFkToUpdExprs map[string]*UpdateExpression + ChildFkToUpdExprs map[string]sqlparser.UpdateExprs FKChecksOff bool } @@ -270,6 +270,64 @@ func (st *SemTable) RemoveNonRequiredForeignKeys(verifyAllFks bool, getAction fu return nil } +func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs) (bool, error) { + // Go over all the update expressions + for _, updateExpr := range updateExprs { + deps := st.RecursiveDeps(updateExpr.Name) + if deps.NumberOfTables() != 1 { + panic("expected to have single table dependency") + } + // Get all the child and parent foreign keys for the given table that the update expression belongs to. + childFks := st.childForeignKeysInvolved[deps] + parentFKs := st.parentForeignKeysInvolved[deps] + + involvedInFk := false + // Check if this updated column is part of any child or parent foreign key. + for _, childFk := range childFks { + if childFk.ParentColumns.FindColumn(updateExpr.Name.Name) >= 0 { + involvedInFk = true + break + } + } + for _, parentFk := range parentFKs { + if parentFk.ChildColumns.FindColumn(updateExpr.Name.Name) >= 0 { + involvedInFk = true + break + } + } + + if !involvedInFk { + continue + } + + dependencyUpdated := false + err := sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { + col, ok := node.(*sqlparser.ColName) + if !ok { + return true, nil + } + // self reference column dependency is not considered a dependent column being updated. + if st.EqualsExpr(updateExpr.Name, col) { + return true, nil + } + for _, updExpr := range updateExprs { + if st.EqualsExpr(updExpr.Name, col) { + dependencyUpdated = true + return false, nil + } + } + return false, nil + }, updateExpr.Expr) + if err != nil { + return false, err + } + if dependencyUpdated { + return true, nil + } + } + return false, nil +} + // isShardScoped checks if the foreign key constraint is shard-scoped or not. It uses the vindex information to make this call. func isShardScoped(pTable *vindexes.Table, cTable *vindexes.Table, pCols sqlparser.Columns, cCols sqlparser.Columns) bool { if !pTable.Keyspace.Sharded { From 64bbabdb949f348271918efe3eb31f17a3996e17 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 30 Oct 2023 11:45:01 +0530 Subject: [PATCH 21/43] feat: fix literal updates clubbed with non-literal updates Signed-off-by: Manan Gupta --- go/test/endtoend/vtgate/foreignkey/fk_test.go | 7 ++ go/vt/vtgate/planbuilder/operators/update.go | 20 +++-- .../testdata/foreignkey_cases.json | 89 +++++++++++++++++++ go/vt/vtgate/semantics/semantic_state.go | 5 -- 4 files changed, 111 insertions(+), 10 deletions(-) diff --git a/go/test/endtoend/vtgate/foreignkey/fk_test.go b/go/test/endtoend/vtgate/foreignkey/fk_test.go index db2894fe75b..6877071990b 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_test.go @@ -872,6 +872,13 @@ func TestFkQueries(t *testing.T) { "insert into fk_t12 (id, col) values (1,1),(2,2),(3,3),(4,4),(5,5)", "update fk_t10 set col = id + null where id = 1", }, + }, { + name: "Multi column foreign key update with one literal and one non-literal update", + queries: []string{ + "insert into fk_multicol_t15 (id, cola, colb) values (1,1,1),(2,2,2)", + "insert into fk_multicol_t16 (id, cola, colb) values (1,1,1),(2,2,2)", + "update fk_multicol_t15 set cola = 3, colb = (id * 2) - 2", + }, }, } diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 1f06261b570..8ab231525ca 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -269,13 +269,14 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, var updFkColOffsets [][2]int // TODO: may only store non-literal update exprs OR store non-literal info along with update expr. ue := ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] - for _, updExpr := range ue { - // TODO: not sure why we append for literals. - offset := [2]int{-1, -1} - if !sqlparser.IsLiteral(updExpr.Expr) { + // If ue has any non-literal update, then we need this. + if hasNonLiteralUpdate(ue) { + for _, updExpr := range ue { + // TODO: not sure why we append for literals. + var offset [2]int offset, selectExprs = addUpdExprToSelect(ctx, updExpr, selectExprs) + updFkColOffsets = append(updFkColOffsets, offset) } - updFkColOffsets = append(updFkColOffsets, offset) } fkChild, err := createFkChildForUpdate(ctx, fk, updStmt, selectOffsets, updFkColOffsets, updatedTable) @@ -297,6 +298,15 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, }, nil } +func hasNonLiteralUpdate(exprs sqlparser.UpdateExprs) bool { + for _, expr := range exprs { + if !sqlparser.IsLiteral(expr.Expr) { + return true + } + } + return false +} + func addColumns(ctx *plancontext.PlanningContext, columns sqlparser.Columns, exprs []sqlparser.SelectExpr) ([]int, []sqlparser.SelectExpr) { var offsets []int selectExprs := exprs diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index 7b6e1fe1ff4..020d384dbfd 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -2179,5 +2179,94 @@ "comment": "updating multiple columns of a fk constraint such that one uses the other", "query": "update u_multicol_tbl3 set cola = id, colb = 5 * (cola + (1 - (cola))) where id = 2", "plan": "VT12001: unsupported: same column referenced in foreign key column update is also updated" + }, + { + "comment": "multicol foreign key updates with one literal and one non-literal update", + "query": "update u_multicol_tbl2 set cola = 2, colb = colc - (2) where id = 7", + "plan": { + "QueryType": "UPDATE", + "Original": "update u_multicol_tbl2 set cola = 2, colb = colc - (2) where id = 7", + "Instructions": { + "OperatorType": "FKVerify", + "Inputs": [ + { + "InputName": "VerifyParent-1", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select 1 from u_multicol_tbl2 left join u_multicol_tbl1 on u_multicol_tbl1.cola = 2 and u_multicol_tbl1.colb = u_multicol_tbl2.colc - 2 where 1 != 1", + "Query": "select 1 from u_multicol_tbl2 left join u_multicol_tbl1 on u_multicol_tbl1.cola = 2 and u_multicol_tbl1.colb = u_multicol_tbl2.colc - 2 where u_multicol_tbl2.colc - 2 is not null and u_multicol_tbl2.id = 7 and u_multicol_tbl1.cola is null and u_multicol_tbl1.colb is null limit 1 lock in share mode", + "Table": "u_multicol_tbl1, u_multicol_tbl2" + }, + { + "InputName": "PostVerify", + "OperatorType": "FkCascade", + "Inputs": [ + { + "InputName": "Selection", + "OperatorType": "Route", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "FieldQuery": "select cola, colb, cola != 2, 2, colb != u_multicol_tbl2.colc - 2, u_multicol_tbl2.colc - 2 from u_multicol_tbl2 where 1 != 1", + "Query": "select cola, colb, cola != 2, 2, colb != u_multicol_tbl2.colc - 2, u_multicol_tbl2.colc - 2 from u_multicol_tbl2 where u_multicol_tbl2.id = 7 for update", + "Table": "u_multicol_tbl2" + }, + { + "InputName": "CascadeChild-1", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "BvName": "fkc_vals", + "Cols": [ + 0, + 1 + ], + "CompExprCols": [ + 2, + 4 + ], + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_multicol_tbl3 set cola = :fkc_upd, colb = :fkc_upd1 where (cola, colb) in ::fkc_vals", + "Table": "u_multicol_tbl3", + "UpdateExprBvNames": [ + "fkc_upd", + "fkc_upd1" + ], + "UpdateExprCols": [ + 3, + 5 + ] + }, + { + "InputName": "Parent", + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "unsharded_fk_allow", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_multicol_tbl2 set cola = 2, colb = u_multicol_tbl2.colc - 2 where u_multicol_tbl2.id = 7", + "Table": "u_multicol_tbl2" + } + ] + } + ] + }, + "TablesUsed": [ + "unsharded_fk_allow.u_multicol_tbl1", + "unsharded_fk_allow.u_multicol_tbl2", + "unsharded_fk_allow.u_multicol_tbl3" + ] + } } ] diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 9a22be80423..0396c2bd6d5 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -141,11 +141,6 @@ type ( ColumnName string } - UpdateExpression struct { - Exprs sqlparser.UpdateExprs - DependencyUpdated bool - } - // SchemaInformation is used tp provide table information from Vschema. SchemaInformation interface { FindTableOrVindex(tablename sqlparser.TableName) (*vindexes.Table, vindexes.Vindex, string, topodatapb.TabletType, key.Destination, error) From c2a3ff26e5b1901fa7bd6bf769dacaa4c5c5ee1d Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 30 Oct 2023 17:40:30 +0530 Subject: [PATCH 22/43] feat: refactor code so that checking for non-literal updates is now a function which we only run when required Signed-off-by: Manan Gupta --- go/vt/vtgate/planbuilder/operators/update.go | 5 ---- .../plancontext/planning_context.go | 1 - go/vt/vtgate/planbuilder/update.go | 6 +++++ go/vt/vtgate/semantics/analyzer.go | 26 +++++++------------ go/vt/vtgate/semantics/analyzer_test.go | 2 +- go/vt/vtgate/semantics/semantic_state.go | 22 +++++++++++++++- 6 files changed, 38 insertions(+), 24 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 8ab231525ca..add52cc95d8 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -184,11 +184,6 @@ func createUpdateOperator(ctx *plancontext.PlanningContext, updStmt *sqlparser.U return nil, vterrors.VT12001("multi shard UPDATE with LIMIT") } - if ctx.SemTable.FKChecksOff { - // We have to run the query with FKChecksOff. - updStmt.Comments = updStmt.Comments.Prepend("/*+ SET_VAR(foreign_key_checks=OFF) */").Parsed() - } - route := &Route{ Source: &Update{ QTable: qt, diff --git a/go/vt/vtgate/planbuilder/plancontext/planning_context.go b/go/vt/vtgate/planbuilder/plancontext/planning_context.go index e6469a3cd75..68ccc95b9fd 100644 --- a/go/vt/vtgate/planbuilder/plancontext/planning_context.go +++ b/go/vt/vtgate/planbuilder/plancontext/planning_context.go @@ -78,7 +78,6 @@ func CreatePlanningContext(stmt sqlparser.Statement, SkipPredicates: map[sqlparser.Expr]any{}, PlannerVersion: version, ReservedArguments: map[sqlparser.Expr]string{}, - VerifyAllFKs: semTable.FKChecksOff, }, nil } diff --git a/go/vt/vtgate/planbuilder/update.go b/go/vt/vtgate/planbuilder/update.go index eced4251ab3..f92701a50a8 100644 --- a/go/vt/vtgate/planbuilder/update.go +++ b/go/vt/vtgate/planbuilder/update.go @@ -46,6 +46,12 @@ func gen4UpdateStmtPlanner( return nil, err } + if ctx.SemTable.HasNonLiteralForeignKeyUpdate(updStmt.Exprs) { + ctx.VerifyAllFKs = true + // We have to run the query with FKChecksOff. + updStmt.Comments = updStmt.Comments.Prepend("/*+ SET_VAR(foreign_key_checks=OFF) */").Parsed() + } + // Remove all the foreign keys that don't require any handling. err = ctx.SemTable.RemoveNonRequiredForeignKeys(ctx.VerifyAllFKs, vindexes.UpdateAction) if err != nil { diff --git a/go/vt/vtgate/semantics/analyzer.go b/go/vt/vtgate/semantics/analyzer.go index c5a22447f63..27fbc082f90 100644 --- a/go/vt/vtgate/semantics/analyzer.go +++ b/go/vt/vtgate/semantics/analyzer.go @@ -108,7 +108,7 @@ func (a *analyzer) newSemTable(statement sqlparser.Statement, coll collations.ID columns[union] = info.exprs } - childFks, parentFks, childFkToUpdExprs, fkChecksOff, err := a.getInvolvedForeignKeys(statement) + childFks, parentFks, childFkToUpdExprs, err := a.getInvolvedForeignKeys(statement) if err != nil { return nil, err } @@ -131,7 +131,6 @@ func (a *analyzer) newSemTable(statement sqlparser.Statement, coll collations.ID childForeignKeysInvolved: childFks, parentForeignKeysInvolved: parentFks, ChildFkToUpdExprs: childFkToUpdExprs, - FKChecksOff: fkChecksOff, }, nil } @@ -319,14 +318,14 @@ func (a *analyzer) noteQuerySignature(node sqlparser.SQLNode) { } // getInvolvedForeignKeys gets the foreign keys that might require taking care off when executing the given statement. -func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs, bool, error) { +func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs, error) { // There are only the DML statements that require any foreign keys handling. switch stmt := statement.(type) { case *sqlparser.Delete: // For DELETE statements, none of the parent foreign keys require handling. // So we collect all the child foreign keys. allChildFks, _, err := a.getAllManagedForeignKeys() - return allChildFks, nil, nil, false, err + return allChildFks, nil, nil, err case *sqlparser.Insert: // For INSERT statements, we have 3 different cases: // 1. REPLACE statement: REPLACE statements are essentially DELETEs and INSERTs rolled into one. @@ -336,33 +335,28 @@ func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[Ta // 3. INSERT with ON DUPLICATE KEY UPDATE: This might trigger an update on the columns specified in the ON DUPLICATE KEY UPDATE clause. allChildFks, allParentFKs, err := a.getAllManagedForeignKeys() if err != nil { - return nil, nil, nil, false, err + return nil, nil, nil, err } if stmt.Action == sqlparser.ReplaceAct { - return allChildFks, allParentFKs, nil, false, nil + return allChildFks, allParentFKs, nil, nil } if len(stmt.OnDup) == 0 { - return nil, allParentFKs, nil, false, nil + return nil, allParentFKs, nil, nil } // If only a certain set of columns are being updated, then there might be some child foreign keys that don't need any consideration since their columns aren't being updated. // So, we filter these child foreign keys out. We can't filter any parent foreign keys because the statement will INSERT a row too, which requires validating all the parent foreign keys. updatedChildFks, _, childFkToUpdExprs := a.filterForeignKeysUsingUpdateExpressions(allChildFks, nil, sqlparser.UpdateExprs(stmt.OnDup)) - return updatedChildFks, allParentFKs, childFkToUpdExprs, false, nil + return updatedChildFks, allParentFKs, childFkToUpdExprs, nil case *sqlparser.Update: // For UPDATE queries we get all the parent and child foreign keys, but we can filter some of them out if the columns that they consist off aren't being updated or are set to NULLs. allChildFks, allParentFks, err := a.getAllManagedForeignKeys() if err != nil { - return nil, nil, nil, false, err + return nil, nil, nil, err } childFks, parentFks, childFkToUpdExprs := a.filterForeignKeysUsingUpdateExpressions(allChildFks, allParentFks, stmt.Exprs) - // TODO: add comment for turning off the foreign_key_checks - fkChecksOff := false - if HasNonLiteral(stmt.Exprs, collectParentFksFromMap(parentFks), collectChildFksFromMap(childFks)) { - fkChecksOff = true - } - return childFks, parentFks, childFkToUpdExprs, fkChecksOff, nil + return childFks, parentFks, childFkToUpdExprs, nil default: - return nil, nil, nil, false, nil + return nil, nil, nil, nil } } diff --git a/go/vt/vtgate/semantics/analyzer_test.go b/go/vt/vtgate/semantics/analyzer_test.go index 1d30dd9e9ac..2a9c8847a85 100644 --- a/go/vt/vtgate/semantics/analyzer_test.go +++ b/go/vt/vtgate/semantics/analyzer_test.go @@ -2000,7 +2000,7 @@ func TestGetInvolvedForeignKeys(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - childFks, parentFks, _, _, err := tt.analyzer.getInvolvedForeignKeys(tt.stmt) + childFks, parentFks, _, err := tt.analyzer.getInvolvedForeignKeys(tt.stmt) if tt.expectedErr != "" { require.EqualError(t, err, tt.expectedErr) return diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 0396c2bd6d5..df6f57f9420 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -133,7 +133,6 @@ type ( childForeignKeysInvolved map[TableSet][]vindexes.ChildFKInfo parentForeignKeysInvolved map[TableSet][]vindexes.ParentFKInfo ChildFkToUpdExprs map[string]sqlparser.UpdateExprs - FKChecksOff bool } columnName struct { @@ -323,6 +322,27 @@ func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs return false, nil } +func (st *SemTable) HasNonLiteralForeignKeyUpdate(updExprs sqlparser.UpdateExprs) bool { + for _, updateExpr := range updExprs { + if sqlparser.IsLiteral(updateExpr.Expr) { + continue + } + parentFks := st.parentForeignKeysInvolved[st.RecursiveDeps(updateExpr.Name)] + for _, parentFk := range parentFks { + if parentFk.ChildColumns.FindColumn(updateExpr.Name.Name) >= 0 { + return true + } + } + childFks := st.childForeignKeysInvolved[st.RecursiveDeps(updateExpr.Name)] + for _, childFk := range childFks { + if childFk.ParentColumns.FindColumn(updateExpr.Name.Name) >= 0 { + return true + } + } + } + return false +} + // isShardScoped checks if the foreign key constraint is shard-scoped or not. It uses the vindex information to make this call. func isShardScoped(pTable *vindexes.Table, cTable *vindexes.Table, pCols sqlparser.Columns, cCols sqlparser.Columns) bool { if !pTable.Keyspace.Sharded { From f3c4d7f8f39e4feb5b8f9564ada6cb58bc2f860c Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Mon, 30 Oct 2023 19:40:57 +0530 Subject: [PATCH 23/43] feat: remove unused code Signed-off-by: Manan Gupta --- go/vt/vtgate/semantics/analyzer.go | 35 ------------------------------ 1 file changed, 35 deletions(-) diff --git a/go/vt/vtgate/semantics/analyzer.go b/go/vt/vtgate/semantics/analyzer.go index 27fbc082f90..7993408da63 100644 --- a/go/vt/vtgate/semantics/analyzer.go +++ b/go/vt/vtgate/semantics/analyzer.go @@ -360,41 +360,6 @@ func (a *analyzer) getInvolvedForeignKeys(statement sqlparser.Statement) (map[Ta } } -func collectParentFksFromMap(parentFkMap map[TableSet][]vindexes.ParentFKInfo) []vindexes.ParentFKInfo { - var parentFks []vindexes.ParentFKInfo - for _, fkInfos := range parentFkMap { - parentFks = append(parentFks, fkInfos...) - } - return parentFks -} - -func collectChildFksFromMap(childFkMap map[TableSet][]vindexes.ChildFKInfo) []vindexes.ChildFKInfo { - var childFks []vindexes.ChildFKInfo - for _, fkInfos := range childFkMap { - childFks = append(childFks, fkInfos...) - } - return childFks -} - -func HasNonLiteral(updExprs sqlparser.UpdateExprs, parentFks []vindexes.ParentFKInfo, childFks []vindexes.ChildFKInfo) bool { - for _, updateExpr := range updExprs { - if sqlparser.IsLiteral(updateExpr.Expr) { - continue - } - for _, parentFk := range parentFks { - if parentFk.ChildColumns.FindColumn(updateExpr.Name.Name) >= 0 { - return true - } - } - for _, childFk := range childFks { - if childFk.ParentColumns.FindColumn(updateExpr.Name.Name) >= 0 { - return true - } - } - } - return false -} - // filterForeignKeysUsingUpdateExpressions filters the child and parent foreign key constraints that don't require any validations/cascades given the updated expressions. func (a *analyzer) filterForeignKeysUsingUpdateExpressions(allChildFks map[TableSet][]vindexes.ChildFKInfo, allParentFks map[TableSet][]vindexes.ParentFKInfo, updExprs sqlparser.UpdateExprs) (map[TableSet][]vindexes.ChildFKInfo, map[TableSet][]vindexes.ParentFKInfo, map[string]sqlparser.UpdateExprs) { if len(allChildFks) == 0 && len(allParentFks) == 0 { From b14dc7b1163622d64266253d33fd0fffc78ff8ab Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 31 Oct 2023 10:37:13 +0530 Subject: [PATCH 24/43] test: fix the timeout for waiting for replication Signed-off-by: Manan Gupta --- go/test/endtoend/vtgate/foreignkey/utils_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/vtgate/foreignkey/utils_test.go b/go/test/endtoend/vtgate/foreignkey/utils_test.go index bb02fbdbcbe..b97e57d685c 100644 --- a/go/test/endtoend/vtgate/foreignkey/utils_test.go +++ b/go/test/endtoend/vtgate/foreignkey/utils_test.go @@ -22,6 +22,7 @@ import ( "math/rand" "strings" "testing" + "time" "github.com/stretchr/testify/require" @@ -224,7 +225,7 @@ func verifyDataIsCorrect(t *testing.T, mcmp utils.MySQLCompare, concurrency int) require.NotNil(t, primaryTab) require.NotNil(t, replicaTab) checkReplicationHealthy(t, replicaTab) - cluster.WaitForReplicationPos(t, primaryTab, replicaTab, true, 60.0) + cluster.WaitForReplicationPos(t, primaryTab, replicaTab, true, 1*time.Minute) primaryConn, err := utils.GetMySQLConn(primaryTab, fmt.Sprintf("vt_%v", keyspace.Name)) require.NoError(t, err) replicaConn, err := utils.GetMySQLConn(replicaTab, fmt.Sprintf("vt_%v", keyspace.Name)) From b7778708acd667120c6a4c71a488cc40fc2847fa Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 31 Oct 2023 11:20:28 +0530 Subject: [PATCH 25/43] test: add fuzzer test for olap queries too Signed-off-by: Manan Gupta --- go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go index 80defdcf0a6..f7d7340d6a4 100644 --- a/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go +++ b/go/test/endtoend/vtgate/foreignkey/fk_fuzz_test.go @@ -37,6 +37,7 @@ type QueryFormat string const ( SQLQueries QueryFormat = "SQLQueries" + OlapSQLQueries QueryFormat = "OlapSQLQueries" PreparedStatmentQueries QueryFormat = "PreparedStatmentQueries" PreparedStatementPacket QueryFormat = "PreparedStatementPacket" ) @@ -93,7 +94,7 @@ func (fz *fuzzer) generateQuery() []string { val := rand.Intn(fz.insertShare + fz.updateShare + fz.deleteShare) if val < fz.insertShare { switch fz.queryFormat { - case SQLQueries: + case OlapSQLQueries, SQLQueries: return []string{fz.generateInsertDMLQuery()} case PreparedStatmentQueries: return fz.getPreparedInsertQueries() @@ -103,7 +104,7 @@ func (fz *fuzzer) generateQuery() []string { } if val < fz.insertShare+fz.updateShare { switch fz.queryFormat { - case SQLQueries: + case OlapSQLQueries, SQLQueries: return []string{fz.generateUpdateDMLQuery()} case PreparedStatmentQueries: return fz.getPreparedUpdateQueries() @@ -112,7 +113,7 @@ func (fz *fuzzer) generateQuery() []string { } } switch fz.queryFormat { - case SQLQueries: + case OlapSQLQueries, SQLQueries: return []string{fz.generateDeleteDMLQuery()} case PreparedStatmentQueries: return fz.getPreparedDeleteQueries() @@ -222,13 +223,16 @@ func (fz *fuzzer) runFuzzerThread(t *testing.T, sharded bool, fuzzerThreadId int _, _ = vitessDb.Exec("use `uks`") } } + if fz.queryFormat == OlapSQLQueries { + _ = utils.Exec(t, mcmp.VtConn, "set workload = olap") + } for { // If fuzzer thread is marked to be stopped, then we should exit this go routine. if fz.shouldStop.Load() == true { return } switch fz.queryFormat { - case SQLQueries, PreparedStatmentQueries: + case OlapSQLQueries, SQLQueries, PreparedStatmentQueries: if fz.generateAndExecuteStatementQuery(t, mcmp) { return } @@ -606,7 +610,7 @@ func TestFkFuzzTest(t *testing.T) { for _, tt := range testcases { for _, testSharded := range []bool{false, true} { - for _, queryFormat := range []QueryFormat{SQLQueries, PreparedStatmentQueries, PreparedStatementPacket} { + for _, queryFormat := range []QueryFormat{OlapSQLQueries, SQLQueries, PreparedStatmentQueries, PreparedStatementPacket} { t.Run(getTestName(tt.name, testSharded)+fmt.Sprintf(" QueryFormat - %v", queryFormat), func(t *testing.T) { mcmp, closer := start(t) defer closer() From 14caa242291db9336a3d4d51e692864f2a2a6752 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 31 Oct 2023 15:24:58 +0530 Subject: [PATCH 26/43] feat: add support for non-literal update in streaming Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/fk_cascade.go | 62 ++++++++++++++++--------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index eccb37388ed..ca5107529fe 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -86,9 +86,9 @@ func (fkc *FkCascade) TryExecute(ctx context.Context, vcursor VCursor, bindVars for _, child := range fkc.Children { if len(child.UpdateExprBvNames) > 0 { - err = fkc.executeNonLiteralUpdateFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child) + err = fkc.executeNonLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, false) } else { - err = fkc.executeLiteralUpdateFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child) + err = fkc.executeLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, false) } if err != nil { return nil, err @@ -99,7 +99,7 @@ func (fkc *FkCascade) TryExecute(ctx context.Context, vcursor VCursor, bindVars return vcursor.ExecutePrimitive(ctx, fkc.Parent, bindVars, wantfields) } -func (fkc *FkCascade) executeLiteralUpdateFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild) error { +func (fkc *FkCascade) executeLiteralExprFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild, isStreaming bool) error { // We create a bindVariable for each Child // that stores the tuple of columns involved in the fk constraint. bv := &querypb.BindVariable{ @@ -117,7 +117,12 @@ func (fkc *FkCascade) executeLiteralUpdateFkChild(ctx context.Context, vcursor V // Since this Primitive is always executed in a transaction, the changes should // be rolled back incase of an error. bindVars[child.BVName] = bv - _, err := vcursor.ExecutePrimitive(ctx, child.Exec, bindVars, wantfields) + var err error + if isStreaming { + err = vcursor.StreamExecutePrimitive(ctx, child.Exec, bindVars, wantfields, func(result *sqltypes.Result) error { return nil }) + } else { + _, err = vcursor.ExecutePrimitive(ctx, child.Exec, bindVars, wantfields) + } if err != nil { return err } @@ -125,7 +130,7 @@ func (fkc *FkCascade) executeLiteralUpdateFkChild(ctx context.Context, vcursor V return nil } -func (fkc *FkCascade) executeNonLiteralUpdateFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild) error { +func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild, isStreaming bool) error { for _, row := range selectionRes.Rows { skipRow := true for _, colIdx := range child.CompExprCols { @@ -164,7 +169,12 @@ func (fkc *FkCascade) executeNonLiteralUpdateFkChild(ctx context.Context, vcurso for idx, updateExprBvName := range child.UpdateExprBvNames { bindVars[updateExprBvName] = sqltypes.ValueBindVariable(row[child.UpdateExprCols[idx]]) } - _, err := vcursor.ExecutePrimitive(ctx, child.Exec, bindVars, wantfields) + var err error + if isStreaming { + err = vcursor.StreamExecutePrimitive(ctx, child.Exec, bindVars, wantfields, func(result *sqltypes.Result) error { return nil }) + } else { + _, err = vcursor.ExecutePrimitive(ctx, child.Exec, bindVars, wantfields) + } if err != nil { return err } @@ -178,49 +188,41 @@ func (fkc *FkCascade) executeNonLiteralUpdateFkChild(ctx context.Context, vcurso // TryStreamExecute implements the Primitive interface. func (fkc *FkCascade) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, callback func(*sqltypes.Result) error) error { - // We create a bindVariable for each Child - // that stores the tuple of columns involved in the fk constraint. - var bindVariables []*querypb.BindVariable - for range fkc.Children { - bindVariables = append(bindVariables, &querypb.BindVariable{ - Type: querypb.Type_TUPLE, - }) - } - - // TODO: add execution for non-literal updates. + var selectionRes *sqltypes.Result // Execute the Selection primitive to find the rows that are going to modified. // This will be used to find the rows that need modification on the children. err := vcursor.StreamExecutePrimitive(ctx, fkc.Selection, bindVars, wantfields, func(result *sqltypes.Result) error { if len(result.Rows) == 0 { return nil } - for idx, child := range fkc.Children { - for _, row := range result.Rows { - var tupleValues []sqltypes.Value - for _, colIdx := range child.Cols { - tupleValues = append(tupleValues, row[colIdx]) - } - bindVariables[idx].Values = append(bindVariables[idx].Values, sqltypes.TupleToProto(tupleValues)) - } + if selectionRes == nil { + selectionRes = result + return nil } + selectionRes.Rows = append(selectionRes.Rows, result.Rows...) return nil }) if err != nil { return err } + // If no rows are to be modified, there is nothing to do. + if selectionRes == nil || len(selectionRes.Rows) == 0 { + return callback(&sqltypes.Result{}) + } + // Execute the child primitive, and bail out incase of failure. // Since this Primitive is always executed in a transaction, the changes should // be rolled back incase of an error. - for idx, child := range fkc.Children { - bindVars[child.BVName] = bindVariables[idx] - err = vcursor.StreamExecutePrimitive(ctx, child.Exec, bindVars, wantfields, func(result *sqltypes.Result) error { - return nil - }) + for _, child := range fkc.Children { + if len(child.UpdateExprBvNames) > 0 { + err = fkc.executeNonLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, true) + } else { + err = fkc.executeLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, true) + } if err != nil { return err } - delete(bindVars, child.BVName) } // All the children are modified successfully, we can now execute the Parent Primitive. From 47ee14d16a8d8b6846228f67997103762c8329ad Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 31 Oct 2023 15:25:53 +0530 Subject: [PATCH 27/43] feat: fix fkVerify to rollback transaction when select returns rows Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/fk_verify.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/go/vt/vtgate/engine/fk_verify.go b/go/vt/vtgate/engine/fk_verify.go index 350aeec59e0..c1a9a606092 100644 --- a/go/vt/vtgate/engine/fk_verify.go +++ b/go/vt/vtgate/engine/fk_verify.go @@ -84,15 +84,19 @@ func (f *FkVerify) TryExecute(ctx context.Context, vcursor VCursor, bindVars map // TryStreamExecute implements the Primitive interface func (f *FkVerify) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, callback func(*sqltypes.Result) error) error { for _, v := range f.Verify { + var rowsErr error err := vcursor.StreamExecutePrimitive(ctx, v.Exec, bindVars, wantfields, func(qr *sqltypes.Result) error { if len(qr.Rows) > 0 { - return getError(v.Typ) + rowsErr = getError(v.Typ) } return nil }) if err != nil { return err } + if rowsErr != nil { + return rowsErr + } } return vcursor.StreamExecutePrimitive(ctx, f.Exec, bindVars, wantfields, callback) } From c40e9c4f0505ff0add1f9d4049807aa2a2d9ceec Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 31 Oct 2023 15:48:44 +0530 Subject: [PATCH 28/43] test: update test output post merge Signed-off-by: Manan Gupta --- go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index cabe08e204c..175c4c268b5 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -1258,12 +1258,12 @@ { "InputName": "VerifyParent-1", "OperatorType": "Limit", - "Count": "INT64(1)", + "Count": "1", "Inputs": [ { "OperatorType": "Projection", "Expressions": [ - "INT64(1) as 1" + "1 as 1" ], "Inputs": [ { From 6e152dfeba4b970dff0462790558264e6e51916f Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 31 Oct 2023 16:07:51 +0530 Subject: [PATCH 29/43] feat: fix serializing of foreign keys Signed-off-by: Manan Gupta --- go/vt/vtgate/semantics/analyzer_test.go | 138 ++++++++++-------- go/vt/vtgate/semantics/semantic_state_test.go | 2 +- go/vt/vtgate/vindexes/foreign_keys.go | 16 +- 3 files changed, 85 insertions(+), 71 deletions(-) diff --git a/go/vt/vtgate/semantics/analyzer_test.go b/go/vt/vtgate/semantics/analyzer_test.go index 2a9c8847a85..c24b463af85 100644 --- a/go/vt/vtgate/semantics/analyzer_test.go +++ b/go/vt/vtgate/semantics/analyzer_test.go @@ -1521,89 +1521,103 @@ func fakeSchemaInfo() *FakeSI { return si } +var parentTbl = &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("parentt"), + Keyspace: &vindexes.Keyspace{ + Name: "ks", + }, +} + var tbl = map[string]TableInfo{ "t0": &RealTable{ Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t0"), Keyspace: &vindexes.Keyspace{Name: "ks"}, ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(nil, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(nil, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), }, ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(nil, []string{"colb"}, []string{"colb"}), - pkInfo(nil, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), }, }, }, "t1": &RealTable{ Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t1"), Keyspace: &vindexes.Keyspace{Name: "ks_unmanaged", Sharded: true}, ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(nil, []string{"cola"}, []string{"cola"}, sqlparser.Restrict), - ckInfo(nil, []string{"cola1", "cola2"}, []string{"ccola1", "ccola2"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"cola"}, []string{"cola"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola1", "cola2"}, []string{"ccola1", "ccola2"}, sqlparser.SetNull), }, }, }, "t2": &RealTable{ Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t2"), Keyspace: &vindexes.Keyspace{Name: "ks"}, }, }, "t3": &RealTable{ Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t3"), Keyspace: &vindexes.Keyspace{Name: "undefined_ks", Sharded: true}, }, }, "t4": &RealTable{ Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t4"), Keyspace: &vindexes.Keyspace{Name: "ks"}, ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(nil, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(nil, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), - ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), - ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), + ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), }, ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(nil, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), - pkInfo(nil, []string{"pcolc"}, []string{"colc"}), - pkInfo(nil, []string{"pcolb", "pcola"}, []string{"colb", "cola"}), - pkInfo(nil, []string{"pcolb"}, []string{"colb"}), - pkInfo(nil, []string{"pcola"}, []string{"cola"}), - pkInfo(nil, []string{"pcolb", "pcolx"}, []string{"colb", "colx"}), + pkInfo(parentTbl, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), + pkInfo(parentTbl, []string{"pcolc"}, []string{"colc"}), + pkInfo(parentTbl, []string{"pcolb", "pcola"}, []string{"colb", "cola"}), + pkInfo(parentTbl, []string{"pcolb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), + pkInfo(parentTbl, []string{"pcolb", "pcolx"}, []string{"colb", "colx"}), }, }, }, "t5": &RealTable{ Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t5"), Keyspace: &vindexes.Keyspace{Name: "ks"}, ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - ckInfo(nil, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), - ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), + ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), }, ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(nil, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), - pkInfo(nil, []string{"pcola"}, []string{"cola"}), - pkInfo(nil, []string{"pcold", "pcolc"}, []string{"cold", "colc"}), - pkInfo(nil, []string{"pcold"}, []string{"cold"}), - pkInfo(nil, []string{"pcold", "pcolx"}, []string{"cold", "colx"}), + pkInfo(parentTbl, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), + pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), + pkInfo(parentTbl, []string{"pcold", "pcolc"}, []string{"cold", "colc"}), + pkInfo(parentTbl, []string{"pcold"}, []string{"cold"}), + pkInfo(parentTbl, []string{"pcold", "pcolx"}, []string{"cold", "colx"}), }, }, }, "t6": &RealTable{ Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t6"), Keyspace: &vindexes.Keyspace{Name: "ks"}, ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(nil, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(nil, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), - ckInfo(nil, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(nil, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), - ckInfo(nil, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), - ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), + ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), }, ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(nil, []string{"colb"}, []string{"colb"}), - pkInfo(nil, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), }, }, }, @@ -1637,14 +1651,14 @@ func TestGetAllManagedForeignKeys(t *testing.T) { }, childFkWanted: map[TableSet][]vindexes.ChildFKInfo{ SingleTableSet(0): { - ckInfo(nil, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(nil, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), }, }, parentFkWanted: map[TableSet][]vindexes.ParentFKInfo{ SingleTableSet(0): { - pkInfo(nil, []string{"colb"}, []string{"colb"}), - pkInfo(nil, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), }, }, }, @@ -1732,12 +1746,12 @@ func TestFilterForeignKeysUsingUpdateExpressions(t *testing.T) { updExprs: updateExprs, childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ SingleTableSet(0): { - ckInfo(nil, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(nil, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), }, SingleTableSet(1): { - ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - ckInfo(nil, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), }, }, parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{}, @@ -1753,11 +1767,11 @@ func TestFilterForeignKeysUsingUpdateExpressions(t *testing.T) { childFksWanted: map[TableSet][]vindexes.ChildFKInfo{}, parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ SingleTableSet(0): { - pkInfo(nil, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), - pkInfo(nil, []string{"pcola"}, []string{"cola"}), + pkInfo(parentTbl, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), + pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), }, SingleTableSet(1): { - pkInfo(nil, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), + pkInfo(parentTbl, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), }, }, }, @@ -1804,8 +1818,8 @@ func TestGetInvolvedForeignKeys(t *testing.T) { }, childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ SingleTableSet(0): { - ckInfo(nil, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(nil, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), }, }, }, @@ -1842,21 +1856,21 @@ func TestGetInvolvedForeignKeys(t *testing.T) { }, childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ SingleTableSet(0): { - ckInfo(nil, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(nil, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), }, SingleTableSet(1): { - ckInfo(nil, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - ckInfo(nil, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), }, }, parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ SingleTableSet(0): { - pkInfo(nil, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), - pkInfo(nil, []string{"pcola"}, []string{"cola"}), + pkInfo(parentTbl, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), + pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), }, SingleTableSet(1): { - pkInfo(nil, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), + pkInfo(parentTbl, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), }, }, }, @@ -1881,14 +1895,14 @@ func TestGetInvolvedForeignKeys(t *testing.T) { }, childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ SingleTableSet(0): { - ckInfo(nil, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(nil, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), }, }, parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ SingleTableSet(0): { - pkInfo(nil, []string{"colb"}, []string{"colb"}), - pkInfo(nil, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), }, }, }, @@ -1914,8 +1928,8 @@ func TestGetInvolvedForeignKeys(t *testing.T) { childFksWanted: nil, parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ SingleTableSet(0): { - pkInfo(nil, []string{"colb"}, []string{"colb"}), - pkInfo(nil, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), }, }, }, @@ -1950,14 +1964,14 @@ func TestGetInvolvedForeignKeys(t *testing.T) { }, childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ SingleTableSet(0): { - ckInfo(nil, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(nil, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), }, }, parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ SingleTableSet(0): { - pkInfo(nil, []string{"colb"}, []string{"colb"}), - pkInfo(nil, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), }, }, }, diff --git a/go/vt/vtgate/semantics/semantic_state_test.go b/go/vt/vtgate/semantics/semantic_state_test.go index ab855322d76..741ec9fc6fd 100644 --- a/go/vt/vtgate/semantics/semantic_state_test.go +++ b/go/vt/vtgate/semantics/semantic_state_test.go @@ -418,7 +418,7 @@ func TestRemoveParentForeignKey(t *testing.T) { }, }, }, - fkToIgnore: "ks.t2child_coldks.t3cold", + fkToIgnore: "ks.t2|child_cold||ks.t3|cold", parentFksWanted: []vindexes.ParentFKInfo{ pkInfo(t3Table, []string{"colb"}, []string{"child_colb"}), pkInfo(t3Table, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}), diff --git a/go/vt/vtgate/vindexes/foreign_keys.go b/go/vt/vtgate/vindexes/foreign_keys.go index db984462b25..74f9ce74844 100644 --- a/go/vt/vtgate/vindexes/foreign_keys.go +++ b/go/vt/vtgate/vindexes/foreign_keys.go @@ -46,13 +46,13 @@ func (fk *ParentFKInfo) MarshalJSON() ([]byte, error) { func (fk *ParentFKInfo) String(childTable *Table) string { var str strings.Builder - str.WriteString(childTable.String()) + str.WriteString(sqlparser.String(childTable.GetTableName())) for _, column := range fk.ChildColumns { - str.WriteString(column.String()) + str.WriteString("|" + sqlparser.String(column)) } - str.WriteString(fk.Table.String()) + str.WriteString("||" + sqlparser.String(fk.Table.GetTableName())) for _, column := range fk.ParentColumns { - str.WriteString(column.String()) + str.WriteString("|" + sqlparser.String(column)) } return str.String() } @@ -91,13 +91,13 @@ func (fk *ChildFKInfo) MarshalJSON() ([]byte, error) { func (fk *ChildFKInfo) String(parentTable *Table) string { var str strings.Builder - str.WriteString(fk.Table.String()) + str.WriteString(sqlparser.String(fk.Table.GetTableName())) for _, column := range fk.ChildColumns { - str.WriteString(column.String()) + str.WriteString("|" + sqlparser.String(column)) } - str.WriteString(parentTable.String()) + str.WriteString("||" + sqlparser.String(parentTable.GetTableName())) for _, column := range fk.ParentColumns { - str.WriteString(column.String()) + str.WriteString("|" + sqlparser.String(column)) } return str.String() } From 16ff17c1da12f2517c3d1c2a121474d811a89459 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 31 Oct 2023 16:15:27 +0530 Subject: [PATCH 30/43] refactor: make childFkToUpdExprs map unexported Signed-off-by: Manan Gupta --- go/vt/vtgate/planbuilder/operators/update.go | 6 +++--- go/vt/vtgate/semantics/analyzer.go | 2 +- go/vt/vtgate/semantics/semantic_state.go | 7 ++++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index add52cc95d8..d8b8975d4d2 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -263,7 +263,7 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, // 2. the new value itself var updFkColOffsets [][2]int // TODO: may only store non-literal update exprs OR store non-literal info along with update expr. - ue := ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] + ue := ctx.SemTable.GetUpdateExpressionsForFk(fk.String(updatedTable)) // If ue has any non-literal update, then we need this. if hasNonLiteralUpdate(ue) { for _, updExpr := range ue { @@ -430,7 +430,7 @@ func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.Chi // The update expressions are the same as the update expressions in the parent update query // with the column names replaced with the child column names. var childUpdateExprs sqlparser.UpdateExprs - for idx, updateExpr := range ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)] { + for idx, updateExpr := range ctx.SemTable.GetUpdateExpressionsForFk(fk.String(updatedTable)) { colIdx := fk.ParentColumns.FindColumn(updateExpr.Name.Name) if colIdx == -1 { continue @@ -490,7 +490,7 @@ func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.Chi // For example, if we are setting `update parent cola = :v1 and colb = :v2`, then on the child, the where condition would look something like this - // `:v1 IS NULL OR :v2 IS NULL OR (child_cola, child_colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL). - compExpr := nullSafeNotInComparison(ctx.SemTable.ChildFkToUpdExprs[fk.String(updatedTable)], fk, updatedTable.GetTableName(), updateExprBvNames) + compExpr := nullSafeNotInComparison(ctx.SemTable.GetUpdateExpressionsForFk(fk.String(updatedTable)), fk, updatedTable.GetTableName(), updateExprBvNames) if compExpr != nil { childWhereExpr = &sqlparser.AndExpr{ Left: childWhereExpr, diff --git a/go/vt/vtgate/semantics/analyzer.go b/go/vt/vtgate/semantics/analyzer.go index 7993408da63..74b0077ad1f 100644 --- a/go/vt/vtgate/semantics/analyzer.go +++ b/go/vt/vtgate/semantics/analyzer.go @@ -130,7 +130,7 @@ func (a *analyzer) newSemTable(statement sqlparser.Statement, coll collations.ID QuerySignature: a.sig, childForeignKeysInvolved: childFks, parentForeignKeysInvolved: parentFks, - ChildFkToUpdExprs: childFkToUpdExprs, + childFkToUpdExprs: childFkToUpdExprs, }, nil } diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index ea396103d0d..7925b31aa83 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -133,7 +133,7 @@ type ( // The map is keyed by the tableset of the table that each of the foreign key belongs to. childForeignKeysInvolved map[TableSet][]vindexes.ChildFKInfo parentForeignKeysInvolved map[TableSet][]vindexes.ParentFKInfo - ChildFkToUpdExprs map[string]sqlparser.UpdateExprs + childFkToUpdExprs map[string]sqlparser.UpdateExprs } columnName struct { @@ -182,6 +182,11 @@ func (st *SemTable) GetParentForeignKeysList() []vindexes.ParentFKInfo { return parentFkInfos } +// GetUpdateExpressionsForFk gets the update expressions for the given serialized foreign key constraint. +func (st *SemTable) GetUpdateExpressionsForFk(foreignKey string) sqlparser.UpdateExprs { + return st.childFkToUpdExprs[foreignKey] +} + // RemoveParentForeignKey removes the given foreign key from the parent foreign keys that sem table stores. func (st *SemTable) RemoveParentForeignKey(fkToIgnore string) error { for ts, fkInfos := range st.parentForeignKeysInvolved { From 252f859781a0395b9bea602eb33a65ea3764cc64 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Tue, 31 Oct 2023 16:57:59 +0530 Subject: [PATCH 31/43] refactor: minor refactors and adding comments Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/fk_cascade.go | 36 +++++++---- go/vt/vtgate/planbuilder/operators/update.go | 65 ++++++++------------ go/vt/vtgate/planbuilder/update.go | 3 +- 3 files changed, 51 insertions(+), 53 deletions(-) diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index ca5107529fe..30c4ac54ae3 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -29,12 +29,17 @@ import ( // FkChild contains the Child Primitive to be executed collecting the values from the Selection Primitive using the column indexes. // BVName is used to pass the value as bind variable to the Child Primitive. type FkChild struct { - BVName string - Cols []int // indexes + // BVName is the bind variable name for the tuple bind variable used in the primitive. + BVName string + // Cols are the indexes of the column that need to be selected from the SELECT query to create the tuple bind variable. + Cols []int + // UpdateExprBvNames is the list of bind variables for non-literal expressions in UPDATES. UpdateExprBvNames []string - UpdateExprCols []int - CompExprCols []int - Exec Primitive + // UpdateExprCols is the list of indexes for non-literal expressions in UPDATES that need to be taken from the SELECT. + UpdateExprCols []int + // CompExprCols is the list of indexes for the comparison in the SELECTS to know if the column is actually being updated or not. + CompExprCols []int + Exec Primitive } // FkCascade is a primitive that implements foreign key cascading using Selection as values required to execute the FkChild Primitives. @@ -85,6 +90,8 @@ func (fkc *FkCascade) TryExecute(ctx context.Context, vcursor VCursor, bindVars } for _, child := range fkc.Children { + // Having non-empty UpdateExprBvNames is an indication that we have an update query with non-literal expressions in it. + // We need to run this query differently because we need to run an update for each row we get back from the SELECT. if len(child.UpdateExprBvNames) > 0 { err = fkc.executeNonLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, false) } else { @@ -100,8 +107,7 @@ func (fkc *FkCascade) TryExecute(ctx context.Context, vcursor VCursor, bindVars } func (fkc *FkCascade) executeLiteralExprFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild, isStreaming bool) error { - // We create a bindVariable for each Child - // that stores the tuple of columns involved in the fk constraint. + // We create a bindVariable that stores the tuple of columns involved in the fk constraint. bv := &querypb.BindVariable{ Type: querypb.Type_TUPLE, } @@ -131,31 +137,37 @@ func (fkc *FkCascade) executeLiteralExprFkChild(ctx context.Context, vcursor VCu } func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild, isStreaming bool) error { + // For each row in the SELECT we need to run the child primitive. for _, row := range selectionRes.Rows { + // First we check if any of the columns is being updated at all. skipRow := true for _, colIdx := range child.CompExprCols { + // The comparison expression in NULL means that the update expression itself was NULL. if row[colIdx].IsNull() { skipRow = false break } + // Now we check if the column has updated or not. hasChanged, err := row[colIdx].ToBool() if err != nil { return err } if hasChanged { + // If any column has changed, then we can't skip this row. + // We need to execute the child primitive. skipRow = false break } } + // If none of the columns have changed, then there is no update to cascade, we can move on. if skipRow { continue } - // We create a bindVariable for each Child - // that stores the tuple of columns involved in the fk constraint. + // We create a bindVariable that stores the tuple of columns involved in the fk constraint. bv := &querypb.BindVariable{ Type: querypb.Type_TUPLE, } - // Create a tuple from each Row. + // Create a tuple from the Row. var tupleValues []sqltypes.Value for _, colIdx := range child.Cols { tupleValues = append(tupleValues, row[colIdx]) @@ -163,9 +175,10 @@ func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor bv.Values = append(bv.Values, sqltypes.TupleToProto(tupleValues)) // Execute the child primitive, and bail out incase of failure. // Since this Primitive is always executed in a transaction, the changes should - // be rolled back incase of an error. + // be rolled back in case of an error. bindVars[child.BVName] = bv + // Next, we need to copy the updated expressions value into the bind variables map. for idx, updateExprBvName := range child.UpdateExprBvNames { bindVars[updateExprBvName] = sqltypes.ValueBindVariable(row[child.UpdateExprCols[idx]]) } @@ -178,6 +191,7 @@ func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor if err != nil { return err } + // Remove the bind variables that have been used and are no longer required. delete(bindVars, child.BVName) for _, updateExprBvName := range child.UpdateExprBvNames { delete(bindVars, updateExprBvName) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index d8b8975d4d2..c6e986c92d0 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -125,6 +125,8 @@ func createOperatorFromUpdate(ctx *plancontext.PlanningContext, updStmt *sqlpars return nil, vterrors.VT12001("update with limit with foreign key constraints") } + // Now we check if any of the foreign key columns that are being udpated have dependencies on other updated columns. + // This is unsafe, and we currently don't support this in Vitess. depUpd, err := ctx.SemTable.IsFkDependentColumnUpdated(updStmt.Exprs) if err != nil { return nil, err @@ -259,15 +261,14 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, selectOffsets, selectExprs = addColumns(ctx, fk.ParentColumns, selectExprs) // If we are updating a foreign key column to a non-literal value then, need information about - // 1. new value is different from the old value - // 2. the new value itself + // 1. whether the new value is different from the old value + // 2. the new value itself. var updFkColOffsets [][2]int - // TODO: may only store non-literal update exprs OR store non-literal info along with update expr. ue := ctx.SemTable.GetUpdateExpressionsForFk(fk.String(updatedTable)) - // If ue has any non-literal update, then we need this. + // We only need to store these offsets and add these expressions to SELECT when there are non-literal updates present. if hasNonLiteralUpdate(ue) { for _, updExpr := range ue { - // TODO: not sure why we append for literals. + // We add the expression and a comparison expression to the SELECT exprssion while storing their offsets. var offset [2]int offset, selectExprs = addUpdExprToSelect(ctx, updExpr, selectExprs) updFkColOffsets = append(updFkColOffsets, offset) @@ -293,6 +294,7 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, }, nil } +// hasNonLiteralUpdate checks if any of the update expressions have a non-literal update. func hasNonLiteralUpdate(exprs sqlparser.UpdateExprs) bool { for _, expr := range exprs { if !sqlparser.IsLiteral(expr.Expr) { @@ -302,6 +304,8 @@ func hasNonLiteralUpdate(exprs sqlparser.UpdateExprs) bool { return false } +// addColumns adds the given set of columns to the select expressions provided. It tries to reuse the columns if already present in it. +// It returns the list of offsets for the columns and the updated select expressions. func addColumns(ctx *plancontext.PlanningContext, columns sqlparser.Columns, exprs []sqlparser.SelectExpr) ([]int, []sqlparser.SelectExpr) { var offsets []int selectExprs := exprs @@ -324,9 +328,14 @@ func addColumns(ctx *plancontext.PlanningContext, columns sqlparser.Columns, exp return offsets, selectExprs } +// For an update query having non-literal updates, we add the updated expression and a comparison expression to the select query. +// For example, for a query like `update fk_table set col = id * 100 + 1` +// We would add the expression `id * 100 + 1` and the comparison expression `col != id * 100 + 1` to the select query. func addUpdExprToSelect(ctx *plancontext.PlanningContext, updExpr *sqlparser.UpdateExpr, exprs []sqlparser.SelectExpr) ([2]int, []sqlparser.SelectExpr) { + // Create the comparison expression. compExpr := sqlparser.NewComparisonExpr(sqlparser.NotEqualOp, updExpr.Name, updExpr.Expr, nil) offsets := [2]int{-1, -1} + // Add the expressions to the select expressions. We make sure to reuse the offset if it has already been added once. for idx, selectExpr := range exprs { if ctx.SemTable.EqualsExpr(selectExpr.(*sqlparser.AliasedExpr).Expr, compExpr) { offsets[0] = idx @@ -335,6 +344,7 @@ func addUpdExprToSelect(ctx *plancontext.PlanningContext, updExpr *sqlparser.Upd offsets[1] = idx } } + // If the expression doesn't exist, then we add the expression and store the offset. if offsets[0] == -1 { offsets[0] = len(exprs) exprs = append(exprs, aeWrap(compExpr)) @@ -346,16 +356,6 @@ func addUpdExprToSelect(ctx *plancontext.PlanningContext, updExpr *sqlparser.Upd return offsets, exprs } -func addExprToSelect(ctx *plancontext.PlanningContext, expr sqlparser.Expr, exprs []sqlparser.SelectExpr) (int, []sqlparser.SelectExpr) { - for idx, selectExpr := range exprs { - if ctx.SemTable.EqualsExpr(selectExpr.(*sqlparser.AliasedExpr).Expr, expr) { - return idx, exprs - } - } - offset := len(exprs) - return offset, append(exprs, aeWrap(expr)) -} - // createFkChildForUpdate creates the update query operator for the child table based on the foreign key constraints. func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, selectOffsets []int, updFkColOffsets [][2]int, updatedTable *vindexes.Table) (*FkChild, error) { // Create a ValTuple of child column names @@ -370,13 +370,11 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF compExpr := sqlparser.NewComparisonExpr(sqlparser.InOp, valTuple, sqlparser.NewListArg(bvName), nil) var childWhereExpr sqlparser.Expr = compExpr + // In the case of non-literal updates, we need to assign bindvariables for storing the updated value of the columns + // coming from the SELECT query. var updateExprBvNames []string if len(updFkColOffsets) > 0 { - for _, offset := range updFkColOffsets { - if offset[1] == -1 { - updateExprBvNames = append(updateExprBvNames, "") - continue - } + for range updFkColOffsets { updateBvName := ctx.ReservedVars.ReserveVariable(foreignKeyUpdateExpr) updateExprBvNames = append(updateExprBvNames, updateBvName) } @@ -396,7 +394,7 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF return nil, err } - updatedOffsets, compOffsets, updateExprBvNames := compressUpdateOffsets(updFkColOffsets, updateExprBvNames) + updatedOffsets, compOffsets := splitUpdateOffsets(updFkColOffsets) return &FkChild{ BVName: bvName, @@ -408,18 +406,14 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF }, nil } -func compressUpdateOffsets(offsets [][2]int, updateExprBvNames []string) ([]int, []int, []string) { +// splitUpdateOffsets splits the slice of a pair of ints, into 2 separate slices of ints. +func splitUpdateOffsets(offsets [][2]int) ([]int, []int) { var newUpdatedOffsets, newCompOffsets []int - var newUpdateExprBvNames []string - for idx, offset := range offsets { - if offset[1] == -1 { - continue - } + for _, offset := range offsets { newUpdatedOffsets = append(newUpdatedOffsets, offset[1]) newCompOffsets = append(newCompOffsets, offset[0]) - newUpdateExprBvNames = append(newUpdateExprBvNames, updateExprBvNames[idx]) } - return newUpdatedOffsets, newCompOffsets, newUpdateExprBvNames + return newUpdatedOffsets, newCompOffsets } // buildChildUpdOpForCascade builds the child update statement operator for the CASCADE type foreign key constraint. @@ -706,7 +700,6 @@ func createFkVerifyOpForChildFKForUpdate(ctx *plancontext.PlanningContext, updSt // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL) // This expression is used in cascading SET NULLs and in verifying whether an update should be restricted. func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.ChildFKInfo, parentTbl sqlparser.TableName, updatedExprBvNames []string) sqlparser.Expr { - //var updateValues sqlparser.ValTuple = make([]sqlparser.Expr, len(cFk.ChildColumns)) var valTuple sqlparser.ValTuple var updateValues sqlparser.ValTuple for idx, updateExpr := range updateExprs { @@ -719,21 +712,11 @@ func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.Chi if len(updatedExprBvNames) > 0 && updatedExprBvNames[idx] != "" { childUpdateExpr = sqlparser.NewArgument(updatedExprBvNames[idx]) } - //updateValues[colIdx] = childUpdateExpr updateValues = append(updateValues, childUpdateExpr) valTuple = append(valTuple, sqlparser.NewColNameWithQualifier(cFk.ChildColumns[colIdx].String(), cFk.Table.GetTableName())) } } - //for idx, value := range updateValues { - // if value == nil { - // updateValues[idx] = sqlparser.NewColNameWithQualifier(cFk.ChildColumns[idx].String(), cFk.Table.GetTableName()) - // } - //} - // Create a ValTuple of child column names - //var valTuple sqlparser.ValTuple - //for _, column := range cFk.ChildColumns { - // valTuple = append(valTuple, sqlparser.NewColNameWithQualifier(column.String(), cFk.Table.GetTableName())) - //} + var finalExpr sqlparser.Expr = sqlparser.NewComparisonExpr(sqlparser.NotInOp, valTuple, sqlparser.ValTuple{updateValues}, nil) for _, value := range updateValues { finalExpr = &sqlparser.OrExpr{ diff --git a/go/vt/vtgate/planbuilder/update.go b/go/vt/vtgate/planbuilder/update.go index f92701a50a8..dfb841a4d11 100644 --- a/go/vt/vtgate/planbuilder/update.go +++ b/go/vt/vtgate/planbuilder/update.go @@ -46,9 +46,10 @@ func gen4UpdateStmtPlanner( return nil, err } + // If there are non-literal foreign key updates, we have to run the query with foreign key checks off. if ctx.SemTable.HasNonLiteralForeignKeyUpdate(updStmt.Exprs) { + // Since we are running the query with foreign key checks off, we have to verify all the foreign keys validity on vtgate. ctx.VerifyAllFKs = true - // We have to run the query with FKChecksOff. updStmt.Comments = updStmt.Comments.Prepend("/*+ SET_VAR(foreign_key_checks=OFF) */").Parsed() } From e1abd98ba2af85a2974141d699794d6587916ea5 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 1 Nov 2023 11:20:59 +0530 Subject: [PATCH 32/43] test: add tests for fk_cascade engine changes Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/fk_cascade_test.go | 72 ++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/go/vt/vtgate/engine/fk_cascade_test.go b/go/vt/vtgate/engine/fk_cascade_test.go index ddd381003b1..e93b2a2d3e3 100644 --- a/go/vt/vtgate/engine/fk_cascade_test.go +++ b/go/vt/vtgate/engine/fk_cascade_test.go @@ -149,6 +149,78 @@ func TestUpdateCascade(t *testing.T) { }) } +// TestNonLiteralUpdateCascade tests that FkCascade executes the child and parent primitives for a non-literal update cascade. +func TestNonLiteralUpdateCascade(t *testing.T) { + fakeRes := sqltypes.MakeTestResult(sqltypes.MakeTestFields("cola|cola != colb + 2|colb + 2", "int64|int64|int64"), "1|0|3", "2|1|5", "3|1|7") + + inputP := &Route{ + Query: "select cola, cola != colb + 2, colb + 2, from parent where foo = 48", + RoutingParameters: &RoutingParameters{ + Opcode: Unsharded, + Keyspace: &vindexes.Keyspace{Name: "ks"}, + }, + } + childP := &Update{ + DML: &DML{ + Query: "update child set ca = :fkc_upd where (ca) in ::__vals", + RoutingParameters: &RoutingParameters{ + Opcode: Unsharded, + Keyspace: &vindexes.Keyspace{Name: "ks"}, + }, + }, + } + parentP := &Update{ + DML: &DML{ + Query: "update parent set cola = colb + 2 where foo = 48", + RoutingParameters: &RoutingParameters{ + Opcode: Unsharded, + Keyspace: &vindexes.Keyspace{Name: "ks"}, + }, + }, + } + fkc := &FkCascade{ + Selection: inputP, + Children: []*FkChild{{ + BVName: "__vals", + Cols: []int{0}, + UpdateExprBvNames: []string{"fkc_upd"}, + UpdateExprCols: []int{2}, + CompExprCols: []int{1}, + Exec: childP, + }}, + Parent: parentP, + } + + vc := newDMLTestVCursor("0") + vc.results = []*sqltypes.Result{fakeRes} + _, err := fkc.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, true) + require.NoError(t, err) + vc.ExpectLog(t, []string{ + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.0: select cola, cola != colb + 2, colb + 2, from parent where foo = 48 {} false false`, + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.0: update child set ca = :fkc_upd where (ca) in ::__vals {__vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x012"} fkc_upd: type:INT64 value:"5"} true true`, + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.0: update child set ca = :fkc_upd where (ca) in ::__vals {__vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x013"} fkc_upd: type:INT64 value:"7"} true true`, + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.0: update parent set cola = colb + 2 where foo = 48 {} true true`, + }) + + vc.Rewind() + err = fkc.TryStreamExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, true, func(result *sqltypes.Result) error { return nil }) + require.NoError(t, err) + vc.ExpectLog(t, []string{ + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `StreamExecuteMulti select cola, cola != colb + 2, colb + 2, from parent where foo = 48 ks.0: {} `, + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.0: update child set ca = :fkc_upd where (ca) in ::__vals {__vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x012"} fkc_upd: type:INT64 value:"5"} true true`, + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.0: update child set ca = :fkc_upd where (ca) in ::__vals {__vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x013"} fkc_upd: type:INT64 value:"7"} true true`, + `ResolveDestinations ks [] Destinations:DestinationAllShards()`, + `ExecuteMultiShard ks.0: update parent set cola = colb + 2 where foo = 48 {} true true`, + }) +} + // TestNeedsTransactionInExecPrepared tests that if we have a foreign key cascade inside an ExecStmt plan, then we do mark the plan to require a transaction. func TestNeedsTransactionInExecPrepared(t *testing.T) { // Even if FkCascade is wrapped in ExecStmt, the plan should be marked such that it requires a transaction. From a758a784f8d5dd0447167a9b4a94f8bbb2d51416 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 1 Nov 2023 15:14:33 +0530 Subject: [PATCH 33/43] test: add tests for the function checking for foreign key columns being dependent on updated columns Signed-off-by: Manan Gupta --- go/vt/vtgate/planbuilder/operators/update.go | 6 +- go/vt/vtgate/semantics/semantic_state.go | 18 +-- go/vt/vtgate/semantics/semantic_state_test.go | 114 ++++++++++++++++++ 3 files changed, 126 insertions(+), 12 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index c6e986c92d0..b9784102398 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -127,11 +127,7 @@ func createOperatorFromUpdate(ctx *plancontext.PlanningContext, updStmt *sqlpars // Now we check if any of the foreign key columns that are being udpated have dependencies on other updated columns. // This is unsafe, and we currently don't support this in Vitess. - depUpd, err := ctx.SemTable.IsFkDependentColumnUpdated(updStmt.Exprs) - if err != nil { - return nil, err - } - if depUpd { + if ctx.SemTable.IsFkDependentColumnUpdated(updStmt.Exprs) { return nil, vterrors.VT12001("same column referenced in foreign key column update is also updated") } diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 7925b31aa83..94b811d6cc1 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -270,7 +270,8 @@ func (st *SemTable) RemoveNonRequiredForeignKeys(verifyAllFks bool, getAction fu return nil } -func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs) (bool, error) { +// IsFkDependentColumnUpdated checks if a foreign key column that is being updated is dependent on another column which also being updated. +func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs) bool { // Go over all the update expressions for _, updateExpr := range updateExprs { deps := st.RecursiveDeps(updateExpr.Name) @@ -300,8 +301,14 @@ func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs continue } + // We cannot support updating a foreign key column that is using a column which is also being updated. + // For 2 reasons— + // 1. For the child foreign keys, we aren't sure what the final value of the updated foreign key column will be. So we don't know + // what to cascade to the child. The selection that we do isn't enough to know if the updated value, since one of the columns used in the update is also being updated. + // 2. For the parent foriegn keys, we don't know if we need to reject this update. Because we don't know the final updated value, the update might be needed to be failed, + // but we can't say for certain. dependencyUpdated := false - err := sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { + _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { col, ok := node.(*sqlparser.ColName) if !ok { return true, nil @@ -318,14 +325,11 @@ func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs } return false, nil }, updateExpr.Expr) - if err != nil { - return false, err - } if dependencyUpdated { - return true, nil + return true } } - return false, nil + return false } func (st *SemTable) HasNonLiteralForeignKeyUpdate(updExprs sqlparser.UpdateExprs) bool { diff --git a/go/vt/vtgate/semantics/semantic_state_test.go b/go/vt/vtgate/semantics/semantic_state_test.go index 741ec9fc6fd..31c5d7c4e95 100644 --- a/go/vt/vtgate/semantics/semantic_state_test.go +++ b/go/vt/vtgate/semantics/semantic_state_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" querypb "vitess.io/vitess/go/vt/proto/query" + vschemapb "vitess.io/vitess/go/vt/proto/vschema" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/vindexes" ) @@ -748,3 +749,116 @@ func TestRemoveNonRequiredForeignKeys(t *testing.T) { }) } } + +func TestIsFkDependentColumnUpdated(t *testing.T) { + keyspaceName := "ks" + t3Table := &vindexes.Table{ + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + Name: sqlparser.NewIdentifierCS("t3"), + } + tests := []struct { + name string + query string + fakeSi *FakeSI + isUpdated bool + }{ + { + name: "updated child foreign key column is dependent on another updated column", + query: "update t1 set col = id + 1, id = 6 where foo = 3", + fakeSi: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + keyspaceName: vschemapb.Keyspace_managed, + }, + Tables: map[string]*vindexes.Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + ChildForeignKeys: []vindexes.ChildFKInfo{ + ckInfo(t3Table, []string{"col"}, []string{"col"}, sqlparser.Cascade), + }, + }, + }, + }, + isUpdated: true, + }, { + name: "updated parent foreign key column is dependent on another updated column", + query: "update t1 set col = id + 1, id = 6 where foo = 3", + fakeSi: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + keyspaceName: vschemapb.Keyspace_managed, + }, + Tables: map[string]*vindexes.Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(t3Table, []string{"col"}, []string{"col"}), + }, + }, + }, + }, + isUpdated: true, + }, { + name: "no foreign key column is dependent on a updated value", + query: "update t1 set col = id + 1 where foo = 3", + fakeSi: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + keyspaceName: vschemapb.Keyspace_managed, + }, + Tables: map[string]*vindexes.Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(t3Table, []string{"col"}, []string{"col"}), + }, + }, + }, + }, + isUpdated: false, + }, { + name: "self-referenced foreign key", + query: "update t1 set col = col + 1 where foo = 3", + fakeSi: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + keyspaceName: vschemapb.Keyspace_managed, + }, + Tables: map[string]*vindexes.Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(t3Table, []string{"col"}, []string{"col"}), + }, + }, + }, + }, + isUpdated: false, + }, { + name: "no foreign keys", + query: "update t1 set col = id + 1, id = 6 where foo = 3", + fakeSi: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + keyspaceName: vschemapb.Keyspace_managed, + }, + Tables: map[string]*vindexes.Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + }, + }, + }, + isUpdated: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stmt, err := sqlparser.Parse(tt.query) + require.NoError(t, err) + semTable, err := Analyze(stmt, keyspaceName, tt.fakeSi) + require.NoError(t, err) + got := semTable.IsFkDependentColumnUpdated(stmt.(*sqlparser.Update).Exprs) + require.EqualValues(t, tt.isUpdated, got) + }) + } +} From fcf6ebae8dad32424267f8ddceb5074c2a2f40ae Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 1 Nov 2023 15:20:07 +0530 Subject: [PATCH 34/43] test: add tests for the function checking for non-literal fk updates Signed-off-by: Manan Gupta --- go/vt/vtgate/semantics/semantic_state.go | 1 + go/vt/vtgate/semantics/semantic_state_test.go | 113 ++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 94b811d6cc1..60d4d6432c6 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -332,6 +332,7 @@ func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs return false } +// HasNonLiteralForeignKeyUpdate returns if any of the updated expressions have a non-literal update and are part of a foreign key. func (st *SemTable) HasNonLiteralForeignKeyUpdate(updExprs sqlparser.UpdateExprs) bool { for _, updateExpr := range updExprs { if sqlparser.IsLiteral(updateExpr.Expr) { diff --git a/go/vt/vtgate/semantics/semantic_state_test.go b/go/vt/vtgate/semantics/semantic_state_test.go index 31c5d7c4e95..02f41655e26 100644 --- a/go/vt/vtgate/semantics/semantic_state_test.go +++ b/go/vt/vtgate/semantics/semantic_state_test.go @@ -862,3 +862,116 @@ func TestIsFkDependentColumnUpdated(t *testing.T) { }) } } + +func TestHasNonLiteralForeignKeyUpdate(t *testing.T) { + keyspaceName := "ks" + t3Table := &vindexes.Table{ + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + Name: sqlparser.NewIdentifierCS("t3"), + } + tests := []struct { + name string + query string + fakeSi *FakeSI + hasNonLiteral bool + }{ + { + name: "non literal child foreign key update", + query: "update t1 set col = id + 1 where foo = 3", + fakeSi: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + keyspaceName: vschemapb.Keyspace_managed, + }, + Tables: map[string]*vindexes.Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + ChildForeignKeys: []vindexes.ChildFKInfo{ + ckInfo(t3Table, []string{"col"}, []string{"col"}, sqlparser.Cascade), + }, + }, + }, + }, + hasNonLiteral: true, + }, { + name: "non literal parent foreign key update", + query: "update t1 set col = id + 1 where foo = 3", + fakeSi: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + keyspaceName: vschemapb.Keyspace_managed, + }, + Tables: map[string]*vindexes.Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(t3Table, []string{"col"}, []string{"col"}), + }, + }, + }, + }, + hasNonLiteral: true, + }, { + name: "literal updates only", + query: "update t1 set col = 1 where foo = 3", + fakeSi: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + keyspaceName: vschemapb.Keyspace_managed, + }, + Tables: map[string]*vindexes.Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(t3Table, []string{"col"}, []string{"col"}), + }, + }, + }, + }, + hasNonLiteral: false, + }, { + name: "self-referenced foreign key", + query: "update t1 set col = col + 1 where foo = 3", + fakeSi: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + keyspaceName: vschemapb.Keyspace_managed, + }, + Tables: map[string]*vindexes.Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(t3Table, []string{"col"}, []string{"col"}), + }, + }, + }, + }, + hasNonLiteral: true, + }, { + name: "no foreign keys", + query: "update t1 set col = id + 1 where foo = 3", + fakeSi: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + keyspaceName: vschemapb.Keyspace_managed, + }, + Tables: map[string]*vindexes.Table{ + "t1": { + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: keyspaceName}, + }, + }, + }, + hasNonLiteral: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stmt, err := sqlparser.Parse(tt.query) + require.NoError(t, err) + semTable, err := Analyze(stmt, keyspaceName, tt.fakeSi) + require.NoError(t, err) + got := semTable.HasNonLiteralForeignKeyUpdate(stmt.(*sqlparser.Update).Exprs) + require.EqualValues(t, tt.hasNonLiteral, got) + }) + } +} From fa8370c770569f6e5c4da98d8d62a217f7d00d7a Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 1 Nov 2023 15:29:10 +0530 Subject: [PATCH 35/43] test: add tests for check fk updated expresssions map Signed-off-by: Manan Gupta --- go/vt/vtgate/semantics/analyzer_test.go | 26 ++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/go/vt/vtgate/semantics/analyzer_test.go b/go/vt/vtgate/semantics/analyzer_test.go index c24b463af85..59c9aa755cb 100644 --- a/go/vt/vtgate/semantics/analyzer_test.go +++ b/go/vt/vtgate/semantics/analyzer_test.go @@ -1792,12 +1792,13 @@ func TestGetInvolvedForeignKeys(t *testing.T) { colc := sqlparser.NewColName("colc") cold := sqlparser.NewColName("cold") tests := []struct { - name string - stmt sqlparser.Statement - analyzer *analyzer - childFksWanted map[TableSet][]vindexes.ChildFKInfo - parentFksWanted map[TableSet][]vindexes.ParentFKInfo - expectedErr string + name string + stmt sqlparser.Statement + analyzer *analyzer + childFksWanted map[TableSet][]vindexes.ChildFKInfo + parentFksWanted map[TableSet][]vindexes.ParentFKInfo + childFkUpdateExprsWanted map[string]sqlparser.UpdateExprs + expectedErr string }{ { name: "Delete Query", @@ -1873,6 +1874,12 @@ func TestGetInvolvedForeignKeys(t *testing.T) { pkInfo(parentTbl, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), }, }, + childFkUpdateExprsWanted: map[string]sqlparser.UpdateExprs{ + "ks.parentt|child_cola|child_colx||ks.t4|cola|colx": {&sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}}, + "ks.parentt|child_colb||ks.t4|colb": {&sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}}, + "ks.parentt|child_colc|child_colx||ks.t5|colc|colx": {&sqlparser.UpdateExpr{Name: colc, Expr: sqlparser.NewIntLiteral("1")}}, + "ks.parentt|child_cold||ks.t5|cold": {&sqlparser.UpdateExpr{Name: cold, Expr: &sqlparser.NullVal{}}}, + }, }, { name: "Replace Query", @@ -1974,6 +1981,10 @@ func TestGetInvolvedForeignKeys(t *testing.T) { pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), }, }, + childFkUpdateExprsWanted: map[string]sqlparser.UpdateExprs{ + "ks.parentt|child_cola|child_colx||ks.t6|cola|colx": {&sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}}, + "ks.parentt|child_colb||ks.t6|colb": {&sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}}, + }, }, { name: "Insert error", @@ -2014,12 +2025,13 @@ func TestGetInvolvedForeignKeys(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - childFks, parentFks, _, err := tt.analyzer.getInvolvedForeignKeys(tt.stmt) + childFks, parentFks, childFkUpdateExprs, err := tt.analyzer.getInvolvedForeignKeys(tt.stmt) if tt.expectedErr != "" { require.EqualError(t, err, tt.expectedErr) return } require.EqualValues(t, tt.childFksWanted, childFks) + require.EqualValues(t, tt.childFkUpdateExprsWanted, childFkUpdateExprs) require.EqualValues(t, tt.parentFksWanted, parentFks) }) } From 8dc17e4298d0bc76bb262823aec0039e966a96de Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 1 Nov 2023 15:35:01 +0530 Subject: [PATCH 36/43] comments: fix typing errors Signed-off-by: Manan Gupta --- go/vt/vtgate/semantics/semantic_state.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 60d4d6432c6..1080abc7ff0 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -301,11 +301,10 @@ func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs continue } - // We cannot support updating a foreign key column that is using a column which is also being updated. - // For 2 reasons— + // We cannot support updating a foreign key column that is using a column which is also being updated for 2 reasons— // 1. For the child foreign keys, we aren't sure what the final value of the updated foreign key column will be. So we don't know // what to cascade to the child. The selection that we do isn't enough to know if the updated value, since one of the columns used in the update is also being updated. - // 2. For the parent foriegn keys, we don't know if we need to reject this update. Because we don't know the final updated value, the update might be needed to be failed, + // 2. For the parent foreign keys, we don't know if we need to reject this update. Because we don't know the final updated value, the update might need to be failed, // but we can't say for certain. dependencyUpdated := false _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { From a74c0aa1c30d47ac75e5359096be3ab6337b9ae2 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 8 Nov 2023 11:28:12 +0530 Subject: [PATCH 37/43] feat: change non-literal comparison added in updates to use null safe equality check Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/fk_cascade.go | 12 ++++------- go/vt/vtgate/planbuilder/operators/update.go | 4 ++-- .../testdata/foreignkey_cases.json | 20 +++++++++---------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index 30c4ac54ae3..79f809b5ee8 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -142,17 +142,13 @@ func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor // First we check if any of the columns is being updated at all. skipRow := true for _, colIdx := range child.CompExprCols { - // The comparison expression in NULL means that the update expression itself was NULL. - if row[colIdx].IsNull() { - skipRow = false - break - } - // Now we check if the column has updated or not. - hasChanged, err := row[colIdx].ToBool() + // We use a null-safe comparison, so the value is guaranteed to be not null. + // We check if the column has updated or not. + isUnchanged, err := row[colIdx].ToBool() if err != nil { return err } - if hasChanged { + if !isUnchanged { // If any column has changed, then we can't skip this row. // We need to execute the child primitive. skipRow = false diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index b9784102398..b2e8bac4be0 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -326,10 +326,10 @@ func addColumns(ctx *plancontext.PlanningContext, columns sqlparser.Columns, exp // For an update query having non-literal updates, we add the updated expression and a comparison expression to the select query. // For example, for a query like `update fk_table set col = id * 100 + 1` -// We would add the expression `id * 100 + 1` and the comparison expression `col != id * 100 + 1` to the select query. +// We would add the expression `id * 100 + 1` and the comparison expression `col <=> id * 100 + 1` to the select query. func addUpdExprToSelect(ctx *plancontext.PlanningContext, updExpr *sqlparser.UpdateExpr, exprs []sqlparser.SelectExpr) ([2]int, []sqlparser.SelectExpr) { // Create the comparison expression. - compExpr := sqlparser.NewComparisonExpr(sqlparser.NotEqualOp, updExpr.Name, updExpr.Expr, nil) + compExpr := sqlparser.NewComparisonExpr(sqlparser.NullSafeEqualOp, updExpr.Name, updExpr.Expr, nil) offsets := [2]int{-1, -1} // Add the expressions to the select expressions. We make sure to reuse the offset if it has already been added once. for idx, selectExpr := range exprs { diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index 175c4c268b5..d0a79d2536e 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -821,8 +821,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select col2, col2 != u_tbl2.col1 + 'bar', u_tbl2.col1 + 'bar' from u_tbl2 where 1 != 1", - "Query": "select col2, col2 != u_tbl2.col1 + 'bar', u_tbl2.col1 + 'bar' from u_tbl2 where u_tbl2.id = 1 for update", + "FieldQuery": "select col2, col2 <=> u_tbl2.col1 + 'bar', u_tbl2.col1 + 'bar' from u_tbl2 where 1 != 1", + "Query": "select col2, col2 <=> u_tbl2.col1 + 'bar', u_tbl2.col1 + 'bar' from u_tbl2 where u_tbl2.id = 1 for update", "Table": "u_tbl2" }, { @@ -890,8 +890,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select col1, col1 != u_tbl1.x + 'bar', u_tbl1.x + 'bar' from u_tbl1 where 1 != 1", - "Query": "select col1, col1 != u_tbl1.x + 'bar', u_tbl1.x + 'bar' from u_tbl1 where id = 1 for update", + "FieldQuery": "select col1, col1 <=> u_tbl1.x + 'bar', u_tbl1.x + 'bar' from u_tbl1 where 1 != 1", + "Query": "select col1, col1 <=> u_tbl1.x + 'bar', u_tbl1.x + 'bar' from u_tbl1 where id = 1 for update", "Table": "u_tbl1" }, { @@ -1991,8 +1991,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select col7, col7 != baz + 1 + col7, baz + 1 + col7 from u_tbl7 where 1 != 1", - "Query": "select col7, col7 != baz + 1 + col7, baz + 1 + col7 from u_tbl7 where bar = 42 for update", + "FieldQuery": "select col7, col7 <=> baz + 1 + col7, baz + 1 + col7 from u_tbl7 where 1 != 1", + "Query": "select col7, col7 <=> baz + 1 + col7, baz + 1 + col7 from u_tbl7 where bar = 42 for update", "Table": "u_tbl7" }, { @@ -2089,8 +2089,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select cola, colb, cola != u_multicol_tbl1.cola + 3, u_multicol_tbl1.cola + 3 from u_multicol_tbl1 where 1 != 1", - "Query": "select cola, colb, cola != u_multicol_tbl1.cola + 3, u_multicol_tbl1.cola + 3 from u_multicol_tbl1 where id = 3 for update", + "FieldQuery": "select cola, colb, cola <=> u_multicol_tbl1.cola + 3, u_multicol_tbl1.cola + 3 from u_multicol_tbl1 where 1 != 1", + "Query": "select cola, colb, cola <=> u_multicol_tbl1.cola + 3, u_multicol_tbl1.cola + 3 from u_multicol_tbl1 where id = 3 for update", "Table": "u_multicol_tbl1" }, { @@ -2213,8 +2213,8 @@ "Name": "unsharded_fk_allow", "Sharded": false }, - "FieldQuery": "select cola, colb, cola != 2, 2, colb != u_multicol_tbl2.colc - 2, u_multicol_tbl2.colc - 2 from u_multicol_tbl2 where 1 != 1", - "Query": "select cola, colb, cola != 2, 2, colb != u_multicol_tbl2.colc - 2, u_multicol_tbl2.colc - 2 from u_multicol_tbl2 where u_multicol_tbl2.id = 7 for update", + "FieldQuery": "select cola, colb, cola <=> 2, 2, colb <=> u_multicol_tbl2.colc - 2, u_multicol_tbl2.colc - 2 from u_multicol_tbl2 where 1 != 1", + "Query": "select cola, colb, cola <=> 2, 2, colb <=> u_multicol_tbl2.colc - 2, u_multicol_tbl2.colc - 2 from u_multicol_tbl2 where u_multicol_tbl2.id = 7 for update", "Table": "u_multicol_tbl2" }, { From b3e1f6ed420e9f195a420522ff2e990176623032 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 8 Nov 2023 11:48:19 +0530 Subject: [PATCH 38/43] feat: change the error to also contain the column names that are causing the query to fail Signed-off-by: Manan Gupta --- go/vt/vtgate/planbuilder/operators/update.go | 4 +-- .../testdata/foreignkey_cases.json | 4 +-- go/vt/vtgate/semantics/semantic_state.go | 14 +++++----- go/vt/vtgate/semantics/semantic_state_test.go | 26 +++++++++++-------- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index b2e8bac4be0..036cf42647c 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -127,8 +127,8 @@ func createOperatorFromUpdate(ctx *plancontext.PlanningContext, updStmt *sqlpars // Now we check if any of the foreign key columns that are being udpated have dependencies on other updated columns. // This is unsafe, and we currently don't support this in Vitess. - if ctx.SemTable.IsFkDependentColumnUpdated(updStmt.Exprs) { - return nil, vterrors.VT12001("same column referenced in foreign key column update is also updated") + if err = ctx.SemTable.ErrIfFkDependentColumnUpdated(updStmt.Exprs); err != nil { + return nil, err } return buildFkOperator(ctx, updOp, updClone, parentFks, childFks, vindexTable) diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index d0a79d2536e..b912c9d6881 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -1972,7 +1972,7 @@ { "comment": "foreign key column updated by using a column which is also getting updated", "query": "update u_tbl1 set foo = 100, col1 = baz + 1 + foo where bar = 42", - "plan": "VT12001: unsupported: same column referenced in foreign key column update is also updated" + "plan": "VT12001: unsupported: foo column referenced in foreign key column col1 is itself updated" }, { "comment": "foreign key column updated by using a column which is also getting updated - self reference column is allowed", @@ -2178,7 +2178,7 @@ { "comment": "updating multiple columns of a fk constraint such that one uses the other", "query": "update u_multicol_tbl3 set cola = id, colb = 5 * (cola + (1 - (cola))) where id = 2", - "plan": "VT12001: unsupported: same column referenced in foreign key column update is also updated" + "plan": "VT12001: unsupported: cola column referenced in foreign key column colb is itself updated" }, { "comment": "multicol foreign key updates with one literal and one non-literal update", diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 74a90a626f8..04633a5048d 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -285,8 +285,8 @@ func (st *SemTable) RemoveNonRequiredForeignKeys(verifyAllFks bool, getAction fu return nil } -// IsFkDependentColumnUpdated checks if a foreign key column that is being updated is dependent on another column which also being updated. -func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs) bool { +// ErrIfFkDependentColumnUpdated checks if a foreign key column that is being updated is dependent on another column which also being updated. +func (st *SemTable) ErrIfFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs) error { // Go over all the update expressions for _, updateExpr := range updateExprs { deps := st.RecursiveDeps(updateExpr.Name) @@ -321,7 +321,7 @@ func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs // what to cascade to the child. The selection that we do isn't enough to know if the updated value, since one of the columns used in the update is also being updated. // 2. For the parent foreign keys, we don't know if we need to reject this update. Because we don't know the final updated value, the update might need to be failed, // but we can't say for certain. - dependencyUpdated := false + var dependencyUpdatedErr error _ = sqlparser.Walk(func(node sqlparser.SQLNode) (kontinue bool, err error) { col, ok := node.(*sqlparser.ColName) if !ok { @@ -333,17 +333,17 @@ func (st *SemTable) IsFkDependentColumnUpdated(updateExprs sqlparser.UpdateExprs } for _, updExpr := range updateExprs { if st.EqualsExpr(updExpr.Name, col) { - dependencyUpdated = true + dependencyUpdatedErr = vterrors.VT12001(fmt.Sprintf("%v column referenced in foreign key column %v is itself updated", sqlparser.String(col), sqlparser.String(updateExpr.Name))) return false, nil } } return false, nil }, updateExpr.Expr) - if dependencyUpdated { - return true + if dependencyUpdatedErr != nil { + return dependencyUpdatedErr } } - return false + return nil } // HasNonLiteralForeignKeyUpdate returns if any of the updated expressions have a non-literal update and are part of a foreign key. diff --git a/go/vt/vtgate/semantics/semantic_state_test.go b/go/vt/vtgate/semantics/semantic_state_test.go index 02f41655e26..b904f3656de 100644 --- a/go/vt/vtgate/semantics/semantic_state_test.go +++ b/go/vt/vtgate/semantics/semantic_state_test.go @@ -757,10 +757,10 @@ func TestIsFkDependentColumnUpdated(t *testing.T) { Name: sqlparser.NewIdentifierCS("t3"), } tests := []struct { - name string - query string - fakeSi *FakeSI - isUpdated bool + name string + query string + fakeSi *FakeSI + updatedErr string }{ { name: "updated child foreign key column is dependent on another updated column", @@ -779,7 +779,7 @@ func TestIsFkDependentColumnUpdated(t *testing.T) { }, }, }, - isUpdated: true, + updatedErr: "VT12001: unsupported: id column referenced in foreign key column col is itself updated", }, { name: "updated parent foreign key column is dependent on another updated column", query: "update t1 set col = id + 1, id = 6 where foo = 3", @@ -797,7 +797,7 @@ func TestIsFkDependentColumnUpdated(t *testing.T) { }, }, }, - isUpdated: true, + updatedErr: "VT12001: unsupported: id column referenced in foreign key column col is itself updated", }, { name: "no foreign key column is dependent on a updated value", query: "update t1 set col = id + 1 where foo = 3", @@ -815,7 +815,7 @@ func TestIsFkDependentColumnUpdated(t *testing.T) { }, }, }, - isUpdated: false, + updatedErr: "", }, { name: "self-referenced foreign key", query: "update t1 set col = col + 1 where foo = 3", @@ -833,7 +833,7 @@ func TestIsFkDependentColumnUpdated(t *testing.T) { }, }, }, - isUpdated: false, + updatedErr: "", }, { name: "no foreign keys", query: "update t1 set col = id + 1, id = 6 where foo = 3", @@ -848,7 +848,7 @@ func TestIsFkDependentColumnUpdated(t *testing.T) { }, }, }, - isUpdated: false, + updatedErr: "", }, } for _, tt := range tests { @@ -857,8 +857,12 @@ func TestIsFkDependentColumnUpdated(t *testing.T) { require.NoError(t, err) semTable, err := Analyze(stmt, keyspaceName, tt.fakeSi) require.NoError(t, err) - got := semTable.IsFkDependentColumnUpdated(stmt.(*sqlparser.Update).Exprs) - require.EqualValues(t, tt.isUpdated, got) + got := semTable.ErrIfFkDependentColumnUpdated(stmt.(*sqlparser.Update).Exprs) + if tt.updatedErr == "" { + require.NoError(t, got) + } else { + require.EqualError(t, got, tt.updatedErr) + } }) } } From 2ffd0ddd8b6bcd488641de676ecdbb136950e7af Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Wed, 8 Nov 2023 12:38:05 +0530 Subject: [PATCH 39/43] feat: refactor code to keep all the non-literal update information together in a struct Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/cached_size.go | 30 +++--- go/vt/vtgate/engine/fk_cascade.go | 43 +++++---- go/vt/vtgate/engine/fk_cascade_test.go | 24 +++-- .../planbuilder/operator_transformers.go | 10 +- .../planbuilder/operators/fk_cascade.go | 21 ++-- go/vt/vtgate/planbuilder/operators/update.go | 85 +++++++--------- .../testdata/foreignkey_cases.json | 96 +++++++++---------- 7 files changed, 148 insertions(+), 161 deletions(-) diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index d791c149d8c..9440a3f364a 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -280,7 +280,7 @@ func (cached *FkChild) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(128) + size += int64(80) } // field BVName string size += hack.RuntimeAllocSize(int64(len(cached.BVName))) @@ -288,21 +288,13 @@ func (cached *FkChild) CachedSize(alloc bool) int64 { { size += hack.RuntimeAllocSize(int64(cap(cached.Cols)) * int64(8)) } - // field UpdateExprBvNames []string + // field NonLiteralInfo []vitess.io/vitess/go/vt/vtgate/engine.NonLiteralUpdateInfo { - size += hack.RuntimeAllocSize(int64(cap(cached.UpdateExprBvNames)) * int64(16)) - for _, elem := range cached.UpdateExprBvNames { - size += hack.RuntimeAllocSize(int64(len(elem))) + size += hack.RuntimeAllocSize(int64(cap(cached.NonLiteralInfo)) * int64(32)) + for _, elem := range cached.NonLiteralInfo { + size += elem.CachedSize(false) } } - // field UpdateExprCols []int - { - size += hack.RuntimeAllocSize(int64(cap(cached.UpdateExprCols)) * int64(8)) - } - // field CompExprCols []int - { - size += hack.RuntimeAllocSize(int64(cap(cached.CompExprCols)) * int64(8)) - } // field Exec vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Exec.(cachedObject); ok { size += cc.CachedSize(true) @@ -627,6 +619,18 @@ func (cached *MergeSort) CachedSize(alloc bool) int64 { } return size } +func (cached *NonLiteralUpdateInfo) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(32) + } + // field UpdateExprBvName string + size += hack.RuntimeAllocSize(int64(len(cached.UpdateExprBvName))) + return size +} func (cached *OnlineDDL) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index 79f809b5ee8..4d246f20bea 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -33,13 +33,20 @@ type FkChild struct { BVName string // Cols are the indexes of the column that need to be selected from the SELECT query to create the tuple bind variable. Cols []int - // UpdateExprBvNames is the list of bind variables for non-literal expressions in UPDATES. - UpdateExprBvNames []string - // UpdateExprCols is the list of indexes for non-literal expressions in UPDATES that need to be taken from the SELECT. - UpdateExprCols []int - // CompExprCols is the list of indexes for the comparison in the SELECTS to know if the column is actually being updated or not. - CompExprCols []int - Exec Primitive + // NonLiteralInfo stores the information that is needed to run an update query with non-literal values. + NonLiteralInfo []NonLiteralUpdateInfo + Exec Primitive +} + +// NonLiteralUpdateInfo stores the information required to process non-literal update queries. +// It stores 3 information- +// 1. UpdateExprCol- The index of the updated expression in the select query. +// 2. UpdateExprBvName- The bind variable name to store the updated expression into. +// 3. CompExprCol- The index of the comparison expression in the select query to know if the row value is actually being changed or not. +type NonLiteralUpdateInfo struct { + UpdateExprCol int + UpdateExprBvName string + CompExprCol int } // FkCascade is a primitive that implements foreign key cascading using Selection as values required to execute the FkChild Primitives. @@ -92,7 +99,7 @@ func (fkc *FkCascade) TryExecute(ctx context.Context, vcursor VCursor, bindVars for _, child := range fkc.Children { // Having non-empty UpdateExprBvNames is an indication that we have an update query with non-literal expressions in it. // We need to run this query differently because we need to run an update for each row we get back from the SELECT. - if len(child.UpdateExprBvNames) > 0 { + if len(child.NonLiteralInfo) > 0 { err = fkc.executeNonLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, false) } else { err = fkc.executeLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, false) @@ -141,10 +148,10 @@ func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor for _, row := range selectionRes.Rows { // First we check if any of the columns is being updated at all. skipRow := true - for _, colIdx := range child.CompExprCols { + for _, info := range child.NonLiteralInfo { // We use a null-safe comparison, so the value is guaranteed to be not null. // We check if the column has updated or not. - isUnchanged, err := row[colIdx].ToBool() + isUnchanged, err := row[info.CompExprCol].ToBool() if err != nil { return err } @@ -175,8 +182,8 @@ func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor bindVars[child.BVName] = bv // Next, we need to copy the updated expressions value into the bind variables map. - for idx, updateExprBvName := range child.UpdateExprBvNames { - bindVars[updateExprBvName] = sqltypes.ValueBindVariable(row[child.UpdateExprCols[idx]]) + for _, info := range child.NonLiteralInfo { + bindVars[info.UpdateExprBvName] = sqltypes.ValueBindVariable(row[info.UpdateExprCol]) } var err error if isStreaming { @@ -189,8 +196,8 @@ func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor } // Remove the bind variables that have been used and are no longer required. delete(bindVars, child.BVName) - for _, updateExprBvName := range child.UpdateExprBvNames { - delete(bindVars, updateExprBvName) + for _, info := range child.NonLiteralInfo { + delete(bindVars, info.UpdateExprBvName) } } return nil @@ -225,7 +232,7 @@ func (fkc *FkCascade) TryStreamExecute(ctx context.Context, vcursor VCursor, bin // Since this Primitive is always executed in a transaction, the changes should // be rolled back incase of an error. for _, child := range fkc.Children { - if len(child.UpdateExprBvNames) > 0 { + if len(child.NonLiteralInfo) > 0 { err = fkc.executeNonLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, true) } else { err = fkc.executeLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, true) @@ -253,10 +260,8 @@ func (fkc *FkCascade) Inputs() ([]Primitive, []map[string]any) { "BvName": child.BVName, "Cols": child.Cols, } - if len(child.UpdateExprBvNames) > 0 { - childInfoMap["UpdateExprBvNames"] = child.UpdateExprBvNames - childInfoMap["UpdateExprCols"] = child.UpdateExprCols - childInfoMap["CompExprCols"] = child.CompExprCols + if len(child.NonLiteralInfo) > 0 { + childInfoMap["NonLiteralUpdateInfo"] = child.NonLiteralInfo } inputsMap = append(inputsMap, childInfoMap) inputs = append(inputs, child.Exec) diff --git a/go/vt/vtgate/engine/fk_cascade_test.go b/go/vt/vtgate/engine/fk_cascade_test.go index e93b2a2d3e3..e6abd21d15f 100644 --- a/go/vt/vtgate/engine/fk_cascade_test.go +++ b/go/vt/vtgate/engine/fk_cascade_test.go @@ -151,10 +151,10 @@ func TestUpdateCascade(t *testing.T) { // TestNonLiteralUpdateCascade tests that FkCascade executes the child and parent primitives for a non-literal update cascade. func TestNonLiteralUpdateCascade(t *testing.T) { - fakeRes := sqltypes.MakeTestResult(sqltypes.MakeTestFields("cola|cola != colb + 2|colb + 2", "int64|int64|int64"), "1|0|3", "2|1|5", "3|1|7") + fakeRes := sqltypes.MakeTestResult(sqltypes.MakeTestFields("cola|cola <=> colb + 2|colb + 2", "int64|int64|int64"), "1|1|3", "2|0|5", "3|0|7") inputP := &Route{ - Query: "select cola, cola != colb + 2, colb + 2, from parent where foo = 48", + Query: "select cola, cola <=> colb + 2, colb + 2, from parent where foo = 48", RoutingParameters: &RoutingParameters{ Opcode: Unsharded, Keyspace: &vindexes.Keyspace{Name: "ks"}, @@ -181,12 +181,16 @@ func TestNonLiteralUpdateCascade(t *testing.T) { fkc := &FkCascade{ Selection: inputP, Children: []*FkChild{{ - BVName: "__vals", - Cols: []int{0}, - UpdateExprBvNames: []string{"fkc_upd"}, - UpdateExprCols: []int{2}, - CompExprCols: []int{1}, - Exec: childP, + BVName: "__vals", + Cols: []int{0}, + NonLiteralInfo: []NonLiteralUpdateInfo{ + { + UpdateExprBvName: "fkc_upd", + UpdateExprCol: 2, + CompExprCol: 1, + }, + }, + Exec: childP, }}, Parent: parentP, } @@ -197,7 +201,7 @@ func TestNonLiteralUpdateCascade(t *testing.T) { require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `ExecuteMultiShard ks.0: select cola, cola != colb + 2, colb + 2, from parent where foo = 48 {} false false`, + `ExecuteMultiShard ks.0: select cola, cola <=> colb + 2, colb + 2, from parent where foo = 48 {} false false`, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.0: update child set ca = :fkc_upd where (ca) in ::__vals {__vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x012"} fkc_upd: type:INT64 value:"5"} true true`, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, @@ -211,7 +215,7 @@ func TestNonLiteralUpdateCascade(t *testing.T) { require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `StreamExecuteMulti select cola, cola != colb + 2, colb + 2, from parent where foo = 48 ks.0: {} `, + `StreamExecuteMulti select cola, cola <=> colb + 2, colb + 2, from parent where foo = 48 ks.0: {} `, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.0: update child set ca = :fkc_upd where (ca) in ::__vals {__vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x012"} fkc_upd: type:INT64 value:"5"} true true`, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 1f3c7df59b8..8f541f88f30 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -139,12 +139,10 @@ func transformFkCascade(ctx *plancontext.PlanningContext, fkc *operators.FkCasca childEngine := childLP.Primitive() children = append(children, &engine.FkChild{ - BVName: child.BVName, - Cols: child.Cols, - UpdateExprBvNames: child.UpdateExprBvNames, - UpdateExprCols: child.UpdateExprCols, - CompExprCols: child.CompExprCols, - Exec: childEngine, + BVName: child.BVName, + Cols: child.Cols, + NonLiteralInfo: child.NonLiteralInfo, + Exec: childEngine, }) } diff --git a/go/vt/vtgate/planbuilder/operators/fk_cascade.go b/go/vt/vtgate/planbuilder/operators/fk_cascade.go index cc764fbcab1..73b902a4980 100644 --- a/go/vt/vtgate/planbuilder/operators/fk_cascade.go +++ b/go/vt/vtgate/planbuilder/operators/fk_cascade.go @@ -19,18 +19,17 @@ package operators import ( "slices" + "vitess.io/vitess/go/vt/vtgate/engine" "vitess.io/vitess/go/vt/vtgate/planbuilder/operators/ops" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" ) // FkChild is used to represent a foreign key child table operation type FkChild struct { - BVName string - Cols []int // indexes - UpdateExprBvNames []string - UpdateExprCols []int - CompExprCols []int - Op ops.Operator + BVName string + Cols []int // indexes + NonLiteralInfo []engine.NonLiteralUpdateInfo + Op ops.Operator noColumns noPredicates @@ -91,12 +90,10 @@ func (fkc *FkCascade) Clone(inputs []ops.Operator) ops.Operator { } newFkc.Children = append(newFkc.Children, &FkChild{ - BVName: fkc.Children[idx-2].BVName, - Cols: slices.Clone(fkc.Children[idx-2].Cols), - UpdateExprCols: slices.Clone(fkc.Children[idx-2].UpdateExprCols), - UpdateExprBvNames: slices.Clone(fkc.Children[idx-2].UpdateExprBvNames), - CompExprCols: slices.Clone(fkc.Children[idx-2].CompExprCols), - Op: operator, + BVName: fkc.Children[idx-2].BVName, + Cols: slices.Clone(fkc.Children[idx-2].Cols), + NonLiteralInfo: slices.Clone(fkc.Children[idx-2].NonLiteralInfo), + Op: operator, }) } return newFkc diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index 036cf42647c..bf8a315ad72 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -259,19 +259,20 @@ func createFKCascadeOp(ctx *plancontext.PlanningContext, parentOp ops.Operator, // If we are updating a foreign key column to a non-literal value then, need information about // 1. whether the new value is different from the old value // 2. the new value itself. - var updFkColOffsets [][2]int + // 3. the bind variable to assign to this value. + var nonLiteralUpdateInfo []engine.NonLiteralUpdateInfo ue := ctx.SemTable.GetUpdateExpressionsForFk(fk.String(updatedTable)) // We only need to store these offsets and add these expressions to SELECT when there are non-literal updates present. if hasNonLiteralUpdate(ue) { for _, updExpr := range ue { // We add the expression and a comparison expression to the SELECT exprssion while storing their offsets. - var offset [2]int - offset, selectExprs = addUpdExprToSelect(ctx, updExpr, selectExprs) - updFkColOffsets = append(updFkColOffsets, offset) + var info engine.NonLiteralUpdateInfo + info, selectExprs = addNonLiteralUpdExprToSelect(ctx, updExpr, selectExprs) + nonLiteralUpdateInfo = append(nonLiteralUpdateInfo, info) } } - fkChild, err := createFkChildForUpdate(ctx, fk, updStmt, selectOffsets, updFkColOffsets, updatedTable) + fkChild, err := createFkChildForUpdate(ctx, fk, selectOffsets, nonLiteralUpdateInfo, updatedTable) if err != nil { return nil, err } @@ -327,33 +328,36 @@ func addColumns(ctx *plancontext.PlanningContext, columns sqlparser.Columns, exp // For an update query having non-literal updates, we add the updated expression and a comparison expression to the select query. // For example, for a query like `update fk_table set col = id * 100 + 1` // We would add the expression `id * 100 + 1` and the comparison expression `col <=> id * 100 + 1` to the select query. -func addUpdExprToSelect(ctx *plancontext.PlanningContext, updExpr *sqlparser.UpdateExpr, exprs []sqlparser.SelectExpr) ([2]int, []sqlparser.SelectExpr) { +func addNonLiteralUpdExprToSelect(ctx *plancontext.PlanningContext, updExpr *sqlparser.UpdateExpr, exprs []sqlparser.SelectExpr) (engine.NonLiteralUpdateInfo, []sqlparser.SelectExpr) { // Create the comparison expression. compExpr := sqlparser.NewComparisonExpr(sqlparser.NullSafeEqualOp, updExpr.Name, updExpr.Expr, nil) - offsets := [2]int{-1, -1} + info := engine.NonLiteralUpdateInfo{ + CompExprCol: -1, + UpdateExprCol: -1, + } // Add the expressions to the select expressions. We make sure to reuse the offset if it has already been added once. for idx, selectExpr := range exprs { if ctx.SemTable.EqualsExpr(selectExpr.(*sqlparser.AliasedExpr).Expr, compExpr) { - offsets[0] = idx + info.CompExprCol = idx } if ctx.SemTable.EqualsExpr(selectExpr.(*sqlparser.AliasedExpr).Expr, updExpr.Expr) { - offsets[1] = idx + info.UpdateExprCol = idx } } // If the expression doesn't exist, then we add the expression and store the offset. - if offsets[0] == -1 { - offsets[0] = len(exprs) + if info.CompExprCol == -1 { + info.CompExprCol = len(exprs) exprs = append(exprs, aeWrap(compExpr)) } - if offsets[1] == -1 { - offsets[1] = len(exprs) + if info.UpdateExprCol == -1 { + info.UpdateExprCol = len(exprs) exprs = append(exprs, aeWrap(updExpr.Expr)) } - return offsets, exprs + return info, exprs } // createFkChildForUpdate creates the update query operator for the child table based on the foreign key constraints. -func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, updStmt *sqlparser.Update, selectOffsets []int, updFkColOffsets [][2]int, updatedTable *vindexes.Table) (*FkChild, error) { +func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, selectOffsets []int, nonLiteralUpdateInfo []engine.NonLiteralUpdateInfo, updatedTable *vindexes.Table) (*FkChild, error) { // Create a ValTuple of child column names var valTuple sqlparser.ValTuple for _, column := range fk.ChildColumns { @@ -368,11 +372,10 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF // In the case of non-literal updates, we need to assign bindvariables for storing the updated value of the columns // coming from the SELECT query. - var updateExprBvNames []string - if len(updFkColOffsets) > 0 { - for range updFkColOffsets { - updateBvName := ctx.ReservedVars.ReserveVariable(foreignKeyUpdateExpr) - updateExprBvNames = append(updateExprBvNames, updateBvName) + if len(nonLiteralUpdateInfo) > 0 { + for idx, info := range nonLiteralUpdateInfo { + info.UpdateExprBvName = ctx.ReservedVars.ReserveVariable(foreignKeyUpdateExpr) + nonLiteralUpdateInfo[idx] = info } } @@ -380,9 +383,9 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF var err error switch fk.OnUpdate { case sqlparser.Cascade: - childOp, err = buildChildUpdOpForCascade(ctx, fk, childWhereExpr, updateExprBvNames, updatedTable) + childOp, err = buildChildUpdOpForCascade(ctx, fk, childWhereExpr, nonLiteralUpdateInfo, updatedTable) case sqlparser.SetNull: - childOp, err = buildChildUpdOpForSetNull(ctx, fk, childWhereExpr, updateExprBvNames, updatedTable) + childOp, err = buildChildUpdOpForSetNull(ctx, fk, childWhereExpr, nonLiteralUpdateInfo, updatedTable) case sqlparser.SetDefault: return nil, vterrors.VT09016() } @@ -390,33 +393,19 @@ func createFkChildForUpdate(ctx *plancontext.PlanningContext, fk vindexes.ChildF return nil, err } - updatedOffsets, compOffsets := splitUpdateOffsets(updFkColOffsets) - return &FkChild{ - BVName: bvName, - Cols: selectOffsets, - Op: childOp, - UpdateExprBvNames: updateExprBvNames, - UpdateExprCols: updatedOffsets, - CompExprCols: compOffsets, + BVName: bvName, + Cols: selectOffsets, + Op: childOp, + NonLiteralInfo: nonLiteralUpdateInfo, }, nil } -// splitUpdateOffsets splits the slice of a pair of ints, into 2 separate slices of ints. -func splitUpdateOffsets(offsets [][2]int) ([]int, []int) { - var newUpdatedOffsets, newCompOffsets []int - for _, offset := range offsets { - newUpdatedOffsets = append(newUpdatedOffsets, offset[1]) - newCompOffsets = append(newCompOffsets, offset[0]) - } - return newUpdatedOffsets, newCompOffsets -} - // buildChildUpdOpForCascade builds the child update statement operator for the CASCADE type foreign key constraint. // The query looks like this - // // `UPDATE SET WHERE IN ()` -func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, childWhereExpr sqlparser.Expr, updatedExprBvNames []string, updatedTable *vindexes.Table) (ops.Operator, error) { +func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, childWhereExpr sqlparser.Expr, nonLiteralUpdateInfo []engine.NonLiteralUpdateInfo, updatedTable *vindexes.Table) (ops.Operator, error) { // The update expressions are the same as the update expressions in the parent update query // with the column names replaced with the child column names. var childUpdateExprs sqlparser.UpdateExprs @@ -429,8 +418,8 @@ func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.Chi // The where condition is the same as the comparison expression above // with the column names replaced with the child column names. childUpdateExpr := updateExpr.Expr - if len(updatedExprBvNames) > 0 && updatedExprBvNames[idx] != "" { - childUpdateExpr = sqlparser.NewArgument(updatedExprBvNames[idx]) + if len(nonLiteralUpdateInfo) > 0 && nonLiteralUpdateInfo[idx].UpdateExprBvName != "" { + childUpdateExpr = sqlparser.NewArgument(nonLiteralUpdateInfo[idx].UpdateExprBvName) } childUpdateExprs = append(childUpdateExprs, &sqlparser.UpdateExpr{ Name: sqlparser.NewColName(fk.ChildColumns[colIdx].String()), @@ -461,7 +450,7 @@ func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.Chi // `UPDATE SET // WHERE IN () // [AND ({ IS NULL OR}... NOT IN ())]` -func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, childWhereExpr sqlparser.Expr, updateExprBvNames []string, updatedTable *vindexes.Table) (ops.Operator, error) { +func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, childWhereExpr sqlparser.Expr, nonLiteralUpdateInfo []engine.NonLiteralUpdateInfo, updatedTable *vindexes.Table) (ops.Operator, error) { // For the SET NULL type constraint, we need to set all the child columns to NULL. var childUpdateExprs sqlparser.UpdateExprs for _, column := range fk.ChildColumns { @@ -480,7 +469,7 @@ func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.Chi // For example, if we are setting `update parent cola = :v1 and colb = :v2`, then on the child, the where condition would look something like this - // `:v1 IS NULL OR :v2 IS NULL OR (child_cola, child_colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL). - compExpr := nullSafeNotInComparison(ctx.SemTable.GetUpdateExpressionsForFk(fk.String(updatedTable)), fk, updatedTable.GetTableName(), updateExprBvNames) + compExpr := nullSafeNotInComparison(ctx.SemTable.GetUpdateExpressionsForFk(fk.String(updatedTable)), fk, updatedTable.GetTableName(), nonLiteralUpdateInfo) if compExpr != nil { childWhereExpr = &sqlparser.AndExpr{ Left: childWhereExpr, @@ -695,7 +684,7 @@ func createFkVerifyOpForChildFKForUpdate(ctx *plancontext.PlanningContext, updSt // `:v1 IS NULL OR :v2 IS NULL OR (cola, colb) NOT IN ((:v1,:v2))` // So, if either of :v1 or :v2 is NULL, then the entire condition is true (which is the same as not having the condition when :v1 or :v2 is NULL) // This expression is used in cascading SET NULLs and in verifying whether an update should be restricted. -func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.ChildFKInfo, parentTbl sqlparser.TableName, updatedExprBvNames []string) sqlparser.Expr { +func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.ChildFKInfo, parentTbl sqlparser.TableName, nonLiteralUpdateInfo []engine.NonLiteralUpdateInfo) sqlparser.Expr { var valTuple sqlparser.ValTuple var updateValues sqlparser.ValTuple for idx, updateExpr := range updateExprs { @@ -705,8 +694,8 @@ func nullSafeNotInComparison(updateExprs sqlparser.UpdateExprs, cFk vindexes.Chi return nil } childUpdateExpr := prefixColNames(parentTbl, updateExpr.Expr) - if len(updatedExprBvNames) > 0 && updatedExprBvNames[idx] != "" { - childUpdateExpr = sqlparser.NewArgument(updatedExprBvNames[idx]) + if len(nonLiteralUpdateInfo) > 0 && nonLiteralUpdateInfo[idx].UpdateExprBvName != "" { + childUpdateExpr = sqlparser.NewArgument(nonLiteralUpdateInfo[idx].UpdateExprBvName) } updateValues = append(updateValues, childUpdateExpr) valTuple = append(valTuple, sqlparser.NewColNameWithQualifier(cFk.ChildColumns[colIdx].String(), cFk.Table.GetTableName())) diff --git a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json index b912c9d6881..afe42a45720 100644 --- a/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/foreignkey_cases.json @@ -838,17 +838,15 @@ "Cols": [ 0 ], - "CompExprCols": [ - 1 + "NonLiteralUpdateInfo": [ + { + "UpdateExprCol": 2, + "UpdateExprBvName": "fkc_upd", + "CompExprCol": 1 + } ], "Query": "update u_tbl3 set col3 = null where (col3) in ::fkc_vals and (:fkc_upd is null or (u_tbl3.col3) not in ((:fkc_upd)))", - "Table": "u_tbl3", - "UpdateExprBvNames": [ - "fkc_upd" - ], - "UpdateExprCols": [ - 2 - ] + "Table": "u_tbl3" }, { "InputName": "Parent", @@ -901,14 +899,12 @@ "Cols": [ 0 ], - "CompExprCols": [ - 1 - ], - "UpdateExprBvNames": [ - "fkc_upd" - ], - "UpdateExprCols": [ - 2 + "NonLiteralUpdateInfo": [ + { + "UpdateExprCol": 2, + "UpdateExprBvName": "fkc_upd", + "CompExprCol": 1 + } ], "Inputs": [ { @@ -960,14 +956,12 @@ "Cols": [ 0 ], - "CompExprCols": [ - 1 - ], - "UpdateExprBvNames": [ - "fkc_upd1" - ], - "UpdateExprCols": [ - 2 + "NonLiteralUpdateInfo": [ + { + "UpdateExprCol": 2, + "UpdateExprBvName": "fkc_upd1", + "CompExprCol": 1 + } ], "Inputs": [ { @@ -2002,14 +1996,12 @@ "Cols": [ 0 ], - "CompExprCols": [ - 1 - ], - "UpdateExprBvNames": [ - "fkc_upd" - ], - "UpdateExprCols": [ - 2 + "NonLiteralUpdateInfo": [ + { + "UpdateExprCol": 2, + "UpdateExprBvName": "fkc_upd", + "CompExprCol": 1 + } ], "Inputs": [ { @@ -2101,14 +2093,12 @@ 0, 1 ], - "CompExprCols": [ - 2 - ], - "UpdateExprBvNames": [ - "fkc_upd" - ], - "UpdateExprCols": [ - 3 + "NonLiteralUpdateInfo": [ + { + "UpdateExprCol": 3, + "UpdateExprBvName": "fkc_upd", + "CompExprCol": 2 + } ], "Inputs": [ { @@ -2231,20 +2221,20 @@ 0, 1 ], - "CompExprCols": [ - 2, - 4 + "NonLiteralUpdateInfo": [ + { + "UpdateExprCol": 3, + "UpdateExprBvName": "fkc_upd", + "CompExprCol": 2 + }, + { + "UpdateExprCol": 5, + "UpdateExprBvName": "fkc_upd1", + "CompExprCol": 4 + } ], "Query": "update /*+ SET_VAR(foreign_key_checks=OFF) */ u_multicol_tbl3 set cola = :fkc_upd, colb = :fkc_upd1 where (cola, colb) in ::fkc_vals", - "Table": "u_multicol_tbl3", - "UpdateExprBvNames": [ - "fkc_upd", - "fkc_upd1" - ], - "UpdateExprCols": [ - 3, - 5 - ] + "Table": "u_multicol_tbl3" }, { "InputName": "Parent", From 0736eff8b5383a3eb1f7b3b638fc128534dd9516 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 10 Nov 2023 10:52:10 +0530 Subject: [PATCH 40/43] feat: remove stream execute from fk_cascade and fk_verify. Instead just use TryExecute instead Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/fk_cascade.go | 38 ++------------------------ go/vt/vtgate/engine/fk_cascade_test.go | 6 ++-- go/vt/vtgate/engine/fk_verify.go | 19 +++---------- go/vt/vtgate/engine/fk_verify_test.go | 6 ++-- 4 files changed, 12 insertions(+), 57 deletions(-) diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index 4d246f20bea..c1bd9c6f486 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -205,45 +205,11 @@ func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor // TryStreamExecute implements the Primitive interface. func (fkc *FkCascade) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, callback func(*sqltypes.Result) error) error { - var selectionRes *sqltypes.Result - // Execute the Selection primitive to find the rows that are going to modified. - // This will be used to find the rows that need modification on the children. - err := vcursor.StreamExecutePrimitive(ctx, fkc.Selection, bindVars, wantfields, func(result *sqltypes.Result) error { - if len(result.Rows) == 0 { - return nil - } - if selectionRes == nil { - selectionRes = result - return nil - } - selectionRes.Rows = append(selectionRes.Rows, result.Rows...) - return nil - }) + res, err := fkc.TryExecute(ctx, vcursor, bindVars, wantfields) if err != nil { return err } - - // If no rows are to be modified, there is nothing to do. - if selectionRes == nil || len(selectionRes.Rows) == 0 { - return callback(&sqltypes.Result{}) - } - - // Execute the child primitive, and bail out incase of failure. - // Since this Primitive is always executed in a transaction, the changes should - // be rolled back incase of an error. - for _, child := range fkc.Children { - if len(child.NonLiteralInfo) > 0 { - err = fkc.executeNonLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, true) - } else { - err = fkc.executeLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, true) - } - if err != nil { - return err - } - } - - // All the children are modified successfully, we can now execute the Parent Primitive. - return vcursor.StreamExecutePrimitive(ctx, fkc.Parent, bindVars, wantfields, callback) + return callback(res) } // Inputs implements the Primitive interface. diff --git a/go/vt/vtgate/engine/fk_cascade_test.go b/go/vt/vtgate/engine/fk_cascade_test.go index e6abd21d15f..942fe44a709 100644 --- a/go/vt/vtgate/engine/fk_cascade_test.go +++ b/go/vt/vtgate/engine/fk_cascade_test.go @@ -80,7 +80,7 @@ func TestDeleteCascade(t *testing.T) { require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `StreamExecuteMulti select cola, colb from parent where foo = 48 ks.0: {} `, + `ExecuteMultiShard ks.0: select cola, colb from parent where foo = 48 {} false false`, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.0: delete from child where (ca, cb) in ::__vals {__vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x011\x950\x01a"} values:{type:TUPLE value:"\x89\x02\x012\x950\x01b"}} true true`, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, @@ -141,7 +141,7 @@ func TestUpdateCascade(t *testing.T) { require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `StreamExecuteMulti select cola, colb from parent where foo = 48 ks.0: {} `, + `ExecuteMultiShard ks.0: select cola, colb from parent where foo = 48 {} false false`, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.0: update child set ca = :vtg1 where (ca, cb) in ::__vals {__vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x011\x950\x01a"} values:{type:TUPLE value:"\x89\x02\x012\x950\x01b"}} true true`, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, @@ -215,7 +215,7 @@ func TestNonLiteralUpdateCascade(t *testing.T) { require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `StreamExecuteMulti select cola, cola <=> colb + 2, colb + 2, from parent where foo = 48 ks.0: {} `, + `ExecuteMultiShard ks.0: select cola, cola <=> colb + 2, colb + 2, from parent where foo = 48 {} false false`, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.0: update child set ca = :fkc_upd where (ca) in ::__vals {__vals: type:TUPLE values:{type:TUPLE value:"\x89\x02\x012"} fkc_upd: type:INT64 value:"5"} true true`, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, diff --git a/go/vt/vtgate/engine/fk_verify.go b/go/vt/vtgate/engine/fk_verify.go index c1a9a606092..8625fbe2362 100644 --- a/go/vt/vtgate/engine/fk_verify.go +++ b/go/vt/vtgate/engine/fk_verify.go @@ -83,22 +83,11 @@ func (f *FkVerify) TryExecute(ctx context.Context, vcursor VCursor, bindVars map // TryStreamExecute implements the Primitive interface func (f *FkVerify) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, callback func(*sqltypes.Result) error) error { - for _, v := range f.Verify { - var rowsErr error - err := vcursor.StreamExecutePrimitive(ctx, v.Exec, bindVars, wantfields, func(qr *sqltypes.Result) error { - if len(qr.Rows) > 0 { - rowsErr = getError(v.Typ) - } - return nil - }) - if err != nil { - return err - } - if rowsErr != nil { - return rowsErr - } + res, err := f.TryExecute(ctx, vcursor, bindVars, wantfields) + if err != nil { + return err } - return vcursor.StreamExecutePrimitive(ctx, f.Exec, bindVars, wantfields, callback) + return callback(res) } // Inputs implements the Primitive interface diff --git a/go/vt/vtgate/engine/fk_verify_test.go b/go/vt/vtgate/engine/fk_verify_test.go index 5635a32bc2c..5c9ff83c2ec 100644 --- a/go/vt/vtgate/engine/fk_verify_test.go +++ b/go/vt/vtgate/engine/fk_verify_test.go @@ -74,7 +74,7 @@ func TestFKVerifyUpdate(t *testing.T) { require.NoError(t, err) vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `StreamExecuteMulti select 1 from child c left join parent p on p.cola = 1 and p.colb = 'a' where p.cola is null and p.colb is null ks.0: {} `, + `ExecuteMultiShard ks.0: select 1 from child c left join parent p on p.cola = 1 and p.colb = 'a' where p.cola is null and p.colb is null {} false false`, `ResolveDestinations ks [] Destinations:DestinationAllShards()`, `ExecuteMultiShard ks.0: update child set cola = 1, colb = 'a' where foo = 48 {} true true`, }) @@ -97,7 +97,7 @@ func TestFKVerifyUpdate(t *testing.T) { require.ErrorContains(t, err, "Cannot add or update a child row: a foreign key constraint fails") vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `StreamExecuteMulti select 1 from child c left join parent p on p.cola = 1 and p.colb = 'a' where p.cola is null and p.colb is null ks.0: {} `, + `ExecuteMultiShard ks.0: select 1 from child c left join parent p on p.cola = 1 and p.colb = 'a' where p.cola is null and p.colb is null {} false false`, }) }) @@ -119,7 +119,7 @@ func TestFKVerifyUpdate(t *testing.T) { require.ErrorContains(t, err, "Cannot delete or update a parent row: a foreign key constraint fails") vc.ExpectLog(t, []string{ `ResolveDestinations ks [] Destinations:DestinationAllShards()`, - `StreamExecuteMulti select 1 from grandchild g join child c on g.cola = c.cola and g.colb = c.colb where c.foo = 48 ks.0: {} `, + `ExecuteMultiShard ks.0: select 1 from grandchild g join child c on g.cola = c.cola and g.colb = c.colb where c.foo = 48 {} false false`, }) }) } From f5ae666c2ed8abd7fe1eb15ceda9f6b1d1540b4b Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 10 Nov 2023 10:53:58 +0530 Subject: [PATCH 41/43] refactor: simplify code Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/fk_cascade.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index c1bd9c6f486..8268dd1736a 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -100,7 +100,7 @@ func (fkc *FkCascade) TryExecute(ctx context.Context, vcursor VCursor, bindVars // Having non-empty UpdateExprBvNames is an indication that we have an update query with non-literal expressions in it. // We need to run this query differently because we need to run an update for each row we get back from the SELECT. if len(child.NonLiteralInfo) > 0 { - err = fkc.executeNonLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, false) + err = fkc.executeNonLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child) } else { err = fkc.executeLiteralExprFkChild(ctx, vcursor, bindVars, wantfields, selectionRes, child, false) } @@ -143,7 +143,7 @@ func (fkc *FkCascade) executeLiteralExprFkChild(ctx context.Context, vcursor VCu return nil } -func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild, isStreaming bool) error { +func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild) error { // For each row in the SELECT we need to run the child primitive. for _, row := range selectionRes.Rows { // First we check if any of the columns is being updated at all. @@ -185,12 +185,7 @@ func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor for _, info := range child.NonLiteralInfo { bindVars[info.UpdateExprBvName] = sqltypes.ValueBindVariable(row[info.UpdateExprCol]) } - var err error - if isStreaming { - err = vcursor.StreamExecutePrimitive(ctx, child.Exec, bindVars, wantfields, func(result *sqltypes.Result) error { return nil }) - } else { - _, err = vcursor.ExecutePrimitive(ctx, child.Exec, bindVars, wantfields) - } + _, err := vcursor.ExecutePrimitive(ctx, child.Exec, bindVars, wantfields) if err != nil { return err } From 4969e504942222de371875651fde2212434f569b Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 10 Nov 2023 11:08:23 +0530 Subject: [PATCH 42/43] refactor: fix comment and refactor code Signed-off-by: Manan Gupta --- go/vt/vtgate/planbuilder/operators/update.go | 16 ++++++++++++++-- go/vt/vtgate/semantics/semantic_state.go | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/update.go b/go/vt/vtgate/planbuilder/operators/update.go index bf8a315ad72..50dfc7e4c16 100644 --- a/go/vt/vtgate/planbuilder/operators/update.go +++ b/go/vt/vtgate/planbuilder/operators/update.go @@ -450,7 +450,13 @@ func buildChildUpdOpForCascade(ctx *plancontext.PlanningContext, fk vindexes.Chi // `UPDATE SET // WHERE IN () // [AND ({ IS NULL OR}... NOT IN ())]` -func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.ChildFKInfo, childWhereExpr sqlparser.Expr, nonLiteralUpdateInfo []engine.NonLiteralUpdateInfo, updatedTable *vindexes.Table) (ops.Operator, error) { +func buildChildUpdOpForSetNull( + ctx *plancontext.PlanningContext, + fk vindexes.ChildFKInfo, + childWhereExpr sqlparser.Expr, + nonLiteralUpdateInfo []engine.NonLiteralUpdateInfo, + updatedTable *vindexes.Table, +) (ops.Operator, error) { // For the SET NULL type constraint, we need to set all the child columns to NULL. var childUpdateExprs sqlparser.UpdateExprs for _, column := range fk.ChildColumns { @@ -485,7 +491,13 @@ func buildChildUpdOpForSetNull(ctx *plancontext.PlanningContext, fk vindexes.Chi } // createFKVerifyOp creates the verify operator for the parent foreign key constraints. -func createFKVerifyOp(ctx *plancontext.PlanningContext, childOp ops.Operator, updStmt *sqlparser.Update, parentFks []vindexes.ParentFKInfo, restrictChildFks []vindexes.ChildFKInfo) (ops.Operator, error) { +func createFKVerifyOp( + ctx *plancontext.PlanningContext, + childOp ops.Operator, + updStmt *sqlparser.Update, + parentFks []vindexes.ParentFKInfo, + restrictChildFks []vindexes.ChildFKInfo, +) (ops.Operator, error) { if len(parentFks) == 0 && len(restrictChildFks) == 0 { return childOp, nil } diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 04633a5048d..48ab4322bc8 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -346,7 +346,7 @@ func (st *SemTable) ErrIfFkDependentColumnUpdated(updateExprs sqlparser.UpdateEx return nil } -// HasNonLiteralForeignKeyUpdate returns if any of the updated expressions have a non-literal update and are part of a foreign key. +// HasNonLiteralForeignKeyUpdate checks for non-literal updates in expressions linked to a foreign key. func (st *SemTable) HasNonLiteralForeignKeyUpdate(updExprs sqlparser.UpdateExprs) bool { for _, updateExpr := range updExprs { if sqlparser.IsLiteral(updateExpr.Expr) { From d319cd65f2d03943258bb94b4421c743d28a0468 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Fri, 10 Nov 2023 11:46:34 +0530 Subject: [PATCH 43/43] feat: refactor code and tests to address review comments Signed-off-by: Manan Gupta --- go/vt/vtgate/engine/fk_cascade.go | 13 +- go/vt/vtgate/semantics/analyzer_fk_test.go | 582 +++++++++++++++++++++ go/vt/vtgate/semantics/analyzer_test.go | 556 -------------------- 3 files changed, 587 insertions(+), 564 deletions(-) create mode 100644 go/vt/vtgate/semantics/analyzer_fk_test.go diff --git a/go/vt/vtgate/engine/fk_cascade.go b/go/vt/vtgate/engine/fk_cascade.go index 8268dd1736a..691e326fec7 100644 --- a/go/vt/vtgate/engine/fk_cascade.go +++ b/go/vt/vtgate/engine/fk_cascade.go @@ -19,6 +19,7 @@ package engine import ( "context" "fmt" + "maps" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" @@ -113,7 +114,8 @@ func (fkc *FkCascade) TryExecute(ctx context.Context, vcursor VCursor, bindVars return vcursor.ExecutePrimitive(ctx, fkc.Parent, bindVars, wantfields) } -func (fkc *FkCascade) executeLiteralExprFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild, isStreaming bool) error { +func (fkc *FkCascade) executeLiteralExprFkChild(ctx context.Context, vcursor VCursor, in map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild, isStreaming bool) error { + bindVars := maps.Clone(in) // We create a bindVariable that stores the tuple of columns involved in the fk constraint. bv := &querypb.BindVariable{ Type: querypb.Type_TUPLE, @@ -139,13 +141,13 @@ func (fkc *FkCascade) executeLiteralExprFkChild(ctx context.Context, vcursor VCu if err != nil { return err } - delete(bindVars, child.BVName) return nil } -func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild) error { +func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor VCursor, in map[string]*querypb.BindVariable, wantfields bool, selectionRes *sqltypes.Result, child *FkChild) error { // For each row in the SELECT we need to run the child primitive. for _, row := range selectionRes.Rows { + bindVars := maps.Clone(in) // First we check if any of the columns is being updated at all. skipRow := true for _, info := range child.NonLiteralInfo { @@ -189,11 +191,6 @@ func (fkc *FkCascade) executeNonLiteralExprFkChild(ctx context.Context, vcursor if err != nil { return err } - // Remove the bind variables that have been used and are no longer required. - delete(bindVars, child.BVName) - for _, info := range child.NonLiteralInfo { - delete(bindVars, info.UpdateExprBvName) - } } return nil } diff --git a/go/vt/vtgate/semantics/analyzer_fk_test.go b/go/vt/vtgate/semantics/analyzer_fk_test.go new file mode 100644 index 00000000000..5ba6041ef5c --- /dev/null +++ b/go/vt/vtgate/semantics/analyzer_fk_test.go @@ -0,0 +1,582 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package semantics + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + vschemapb "vitess.io/vitess/go/vt/proto/vschema" + "vitess.io/vitess/go/vt/sqlparser" + "vitess.io/vitess/go/vt/vtgate/vindexes" +) + +var parentTbl = &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("parentt"), + Keyspace: &vindexes.Keyspace{ + Name: "ks", + }, +} + +var tbl = map[string]TableInfo{ + "t0": &RealTable{ + Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t0"), + Keyspace: &vindexes.Keyspace{Name: "ks"}, + ChildForeignKeys: []vindexes.ChildFKInfo{ + ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + }, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + }, + }, + }, + "t1": &RealTable{ + Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t1"), + Keyspace: &vindexes.Keyspace{Name: "ks_unmanaged", Sharded: true}, + ChildForeignKeys: []vindexes.ChildFKInfo{ + ckInfo(parentTbl, []string{"cola"}, []string{"cola"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola1", "cola2"}, []string{"ccola1", "ccola2"}, sqlparser.SetNull), + }, + }, + }, + "t2": &RealTable{ + Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t2"), + Keyspace: &vindexes.Keyspace{Name: "ks"}, + }, + }, + "t3": &RealTable{ + Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t3"), + Keyspace: &vindexes.Keyspace{Name: "undefined_ks", Sharded: true}, + }, + }, + "t4": &RealTable{ + Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t4"), + Keyspace: &vindexes.Keyspace{Name: "ks"}, + ChildForeignKeys: []vindexes.ChildFKInfo{ + ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), + ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + }, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(parentTbl, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), + pkInfo(parentTbl, []string{"pcolc"}, []string{"colc"}), + pkInfo(parentTbl, []string{"pcolb", "pcola"}, []string{"colb", "cola"}), + pkInfo(parentTbl, []string{"pcolb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), + pkInfo(parentTbl, []string{"pcolb", "pcolx"}, []string{"colb", "colx"}), + }, + }, + }, + "t5": &RealTable{ + Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t5"), + Keyspace: &vindexes.Keyspace{Name: "ks"}, + ChildForeignKeys: []vindexes.ChildFKInfo{ + ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), + }, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(parentTbl, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), + pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), + pkInfo(parentTbl, []string{"pcold", "pcolc"}, []string{"cold", "colc"}), + pkInfo(parentTbl, []string{"pcold"}, []string{"cold"}), + pkInfo(parentTbl, []string{"pcold", "pcolx"}, []string{"cold", "colx"}), + }, + }, + }, + "t6": &RealTable{ + Table: &vindexes.Table{ + Name: sqlparser.NewIdentifierCS("t6"), + Keyspace: &vindexes.Keyspace{Name: "ks"}, + ChildForeignKeys: []vindexes.ChildFKInfo{ + ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + ckInfo(parentTbl, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), + ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + }, + ParentForeignKeys: []vindexes.ParentFKInfo{ + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + }, + }, + }, +} + +// TestGetAllManagedForeignKeys tests the functionality of getAllManagedForeignKeys. +func TestGetAllManagedForeignKeys(t *testing.T) { + tests := []struct { + name string + analyzer *analyzer + childFkWanted map[TableSet][]vindexes.ChildFKInfo + parentFkWanted map[TableSet][]vindexes.ParentFKInfo + expectedErr string + }{ + { + name: "Collect all foreign key constraints", + analyzer: &analyzer{ + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t0"], + tbl["t1"], + &DerivedTable{}, + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + "ks_unmanaged": vschemapb.Keyspace_unmanaged, + }, + }, + }, + }, + childFkWanted: map[TableSet][]vindexes.ChildFKInfo{ + SingleTableSet(0): { + ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + }, + }, + parentFkWanted: map[TableSet][]vindexes.ParentFKInfo{ + SingleTableSet(0): { + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + }, + }, + }, + { + name: "keyspace not found in schema information", + analyzer: &analyzer{ + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t2"], + tbl["t3"], + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + }, + }, + }, + }, + expectedErr: "undefined_ks keyspace not found", + }, + { + name: "Cyclic fk constraints error", + analyzer: &analyzer{ + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t0"], tbl["t1"], + &DerivedTable{}, + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + "ks_unmanaged": vschemapb.Keyspace_unmanaged, + }, + KsError: map[string]error{ + "ks": fmt.Errorf("VT09019: ks has cyclic foreign keys"), + }, + }, + }, + }, + expectedErr: "VT09019: ks has cyclic foreign keys", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + childFk, parentFk, err := tt.analyzer.getAllManagedForeignKeys() + if tt.expectedErr != "" { + require.EqualError(t, err, tt.expectedErr) + return + } + require.EqualValues(t, tt.childFkWanted, childFk) + require.EqualValues(t, tt.parentFkWanted, parentFk) + }) + } +} + +// TestFilterForeignKeysUsingUpdateExpressions tests the functionality of filterForeignKeysUsingUpdateExpressions. +func TestFilterForeignKeysUsingUpdateExpressions(t *testing.T) { + cola := sqlparser.NewColName("cola") + colb := sqlparser.NewColName("colb") + colc := sqlparser.NewColName("colc") + cold := sqlparser.NewColName("cold") + a := &analyzer{ + binder: &binder{ + direct: map[sqlparser.Expr]TableSet{ + cola: SingleTableSet(0), + colb: SingleTableSet(0), + colc: SingleTableSet(1), + cold: SingleTableSet(1), + }, + }, + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t4"], + tbl["t5"], + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + }, + }, + }, + } + updateExprs := sqlparser.UpdateExprs{ + &sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}, + &sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}, + &sqlparser.UpdateExpr{Name: colc, Expr: sqlparser.NewIntLiteral("1")}, + &sqlparser.UpdateExpr{Name: cold, Expr: &sqlparser.NullVal{}}, + } + tests := []struct { + name string + analyzer *analyzer + allChildFks map[TableSet][]vindexes.ChildFKInfo + allParentFks map[TableSet][]vindexes.ParentFKInfo + updExprs sqlparser.UpdateExprs + childFksWanted map[TableSet][]vindexes.ChildFKInfo + parentFksWanted map[TableSet][]vindexes.ParentFKInfo + }{ + { + name: "Child Foreign Keys Filtering", + analyzer: a, + allParentFks: nil, + allChildFks: map[TableSet][]vindexes.ChildFKInfo{ + SingleTableSet(0): tbl["t4"].(*RealTable).Table.ChildForeignKeys, + SingleTableSet(1): tbl["t5"].(*RealTable).Table.ChildForeignKeys, + }, + updExprs: updateExprs, + childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ + SingleTableSet(0): { + ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + }, + SingleTableSet(1): { + ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), + }, + }, + parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{}, + }, { + name: "Parent Foreign Keys Filtering", + analyzer: a, + allParentFks: map[TableSet][]vindexes.ParentFKInfo{ + SingleTableSet(0): tbl["t4"].(*RealTable).Table.ParentForeignKeys, + SingleTableSet(1): tbl["t5"].(*RealTable).Table.ParentForeignKeys, + }, + allChildFks: nil, + updExprs: updateExprs, + childFksWanted: map[TableSet][]vindexes.ChildFKInfo{}, + parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ + SingleTableSet(0): { + pkInfo(parentTbl, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), + pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), + }, + SingleTableSet(1): { + pkInfo(parentTbl, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + childFks, parentFks, _ := tt.analyzer.filterForeignKeysUsingUpdateExpressions(tt.allChildFks, tt.allParentFks, tt.updExprs) + require.EqualValues(t, tt.childFksWanted, childFks) + require.EqualValues(t, tt.parentFksWanted, parentFks) + }) + } +} + +// TestGetInvolvedForeignKeys tests the functionality of getInvolvedForeignKeys. +func TestGetInvolvedForeignKeys(t *testing.T) { + cola := sqlparser.NewColName("cola") + colb := sqlparser.NewColName("colb") + colc := sqlparser.NewColName("colc") + cold := sqlparser.NewColName("cold") + tests := []struct { + name string + stmt sqlparser.Statement + analyzer *analyzer + childFksWanted map[TableSet][]vindexes.ChildFKInfo + parentFksWanted map[TableSet][]vindexes.ParentFKInfo + childFkUpdateExprsWanted map[string]sqlparser.UpdateExprs + expectedErr string + }{ + { + name: "Delete Query", + stmt: &sqlparser.Delete{}, + analyzer: &analyzer{ + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t0"], + tbl["t1"], + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + "ks_unmanaged": vschemapb.Keyspace_unmanaged, + }, + }, + }, + }, + childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ + SingleTableSet(0): { + ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + }, + }, + }, + { + name: "Update statement", + stmt: &sqlparser.Update{ + Exprs: sqlparser.UpdateExprs{ + &sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}, + &sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}, + &sqlparser.UpdateExpr{Name: colc, Expr: sqlparser.NewIntLiteral("1")}, + &sqlparser.UpdateExpr{Name: cold, Expr: &sqlparser.NullVal{}}, + }, + }, + analyzer: &analyzer{ + binder: &binder{ + direct: map[sqlparser.Expr]TableSet{ + cola: SingleTableSet(0), + colb: SingleTableSet(0), + colc: SingleTableSet(1), + cold: SingleTableSet(1), + }, + }, + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t4"], + tbl["t5"], + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + }, + }, + }, + }, + childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ + SingleTableSet(0): { + ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + }, + SingleTableSet(1): { + ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), + }, + }, + parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ + SingleTableSet(0): { + pkInfo(parentTbl, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), + pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), + }, + SingleTableSet(1): { + pkInfo(parentTbl, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), + }, + }, + childFkUpdateExprsWanted: map[string]sqlparser.UpdateExprs{ + "ks.parentt|child_cola|child_colx||ks.t4|cola|colx": {&sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}}, + "ks.parentt|child_colb||ks.t4|colb": {&sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}}, + "ks.parentt|child_colc|child_colx||ks.t5|colc|colx": {&sqlparser.UpdateExpr{Name: colc, Expr: sqlparser.NewIntLiteral("1")}}, + "ks.parentt|child_cold||ks.t5|cold": {&sqlparser.UpdateExpr{Name: cold, Expr: &sqlparser.NullVal{}}}, + }, + }, + { + name: "Replace Query", + stmt: &sqlparser.Insert{ + Action: sqlparser.ReplaceAct, + }, + analyzer: &analyzer{ + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t0"], + tbl["t1"], + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + "ks_unmanaged": vschemapb.Keyspace_unmanaged, + }, + }, + }, + }, + childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ + SingleTableSet(0): { + ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), + }, + }, + parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ + SingleTableSet(0): { + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + }, + }, + }, + { + name: "Insert Query", + stmt: &sqlparser.Insert{ + Action: sqlparser.InsertAct, + }, + analyzer: &analyzer{ + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t0"], + tbl["t1"], + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + "ks_unmanaged": vschemapb.Keyspace_unmanaged, + }, + }, + }, + }, + childFksWanted: nil, + parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ + SingleTableSet(0): { + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + }, + }, + }, + { + name: "Insert Query with On Duplicate", + stmt: &sqlparser.Insert{ + Action: sqlparser.InsertAct, + OnDup: sqlparser.OnDup{ + &sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}, + &sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}, + }, + }, + analyzer: &analyzer{ + binder: &binder{ + direct: map[sqlparser.Expr]TableSet{ + cola: SingleTableSet(0), + colb: SingleTableSet(0), + }, + }, + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t6"], + tbl["t1"], + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + "ks_unmanaged": vschemapb.Keyspace_unmanaged, + }, + }, + }, + }, + childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ + SingleTableSet(0): { + ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), + ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), + }, + }, + parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ + SingleTableSet(0): { + pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), + pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), + }, + }, + childFkUpdateExprsWanted: map[string]sqlparser.UpdateExprs{ + "ks.parentt|child_cola|child_colx||ks.t6|cola|colx": {&sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}}, + "ks.parentt|child_colb||ks.t6|colb": {&sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}}, + }, + }, + { + name: "Insert error", + stmt: &sqlparser.Insert{}, + analyzer: &analyzer{ + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t2"], + tbl["t3"], + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + }, + }, + }, + }, + expectedErr: "undefined_ks keyspace not found", + }, + { + name: "Update error", + stmt: &sqlparser.Update{}, + analyzer: &analyzer{ + tables: &tableCollector{ + Tables: []TableInfo{ + tbl["t2"], + tbl["t3"], + }, + si: &FakeSI{ + KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ + "ks": vschemapb.Keyspace_managed, + }, + }, + }, + }, + expectedErr: "undefined_ks keyspace not found", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + childFks, parentFks, childFkUpdateExprs, err := tt.analyzer.getInvolvedForeignKeys(tt.stmt) + if tt.expectedErr != "" { + require.EqualError(t, err, tt.expectedErr) + return + } + require.EqualValues(t, tt.childFksWanted, childFks) + require.EqualValues(t, tt.childFkUpdateExprsWanted, childFkUpdateExprs) + require.EqualValues(t, tt.parentFksWanted, parentFks) + }) + } +} + +func ckInfo(cTable *vindexes.Table, pCols []string, cCols []string, refAction sqlparser.ReferenceAction) vindexes.ChildFKInfo { + return vindexes.ChildFKInfo{ + Table: cTable, + ParentColumns: sqlparser.MakeColumns(pCols...), + ChildColumns: sqlparser.MakeColumns(cCols...), + OnDelete: refAction, + } +} + +func pkInfo(parentTable *vindexes.Table, pCols []string, cCols []string) vindexes.ParentFKInfo { + return vindexes.ParentFKInfo{ + Table: parentTable, + ParentColumns: sqlparser.MakeColumns(pCols...), + ChildColumns: sqlparser.MakeColumns(cCols...), + } +} diff --git a/go/vt/vtgate/semantics/analyzer_test.go b/go/vt/vtgate/semantics/analyzer_test.go index ea4ebc85899..cb19d5deafd 100644 --- a/go/vt/vtgate/semantics/analyzer_test.go +++ b/go/vt/vtgate/semantics/analyzer_test.go @@ -17,7 +17,6 @@ limitations under the License. package semantics import ( - "fmt" "testing" "github.com/stretchr/testify/assert" @@ -25,7 +24,6 @@ import ( "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" - vschemapb "vitess.io/vitess/go/vt/proto/vschema" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/vindexes" ) @@ -1627,557 +1625,3 @@ func fakeSchemaInfo() *FakeSI { } return si } - -var parentTbl = &vindexes.Table{ - Name: sqlparser.NewIdentifierCS("parentt"), - Keyspace: &vindexes.Keyspace{ - Name: "ks", - }, -} - -var tbl = map[string]TableInfo{ - "t0": &RealTable{ - Table: &vindexes.Table{ - Name: sqlparser.NewIdentifierCS("t0"), - Keyspace: &vindexes.Keyspace{Name: "ks"}, - ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), - }, - ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), - pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), - }, - }, - }, - "t1": &RealTable{ - Table: &vindexes.Table{ - Name: sqlparser.NewIdentifierCS("t1"), - Keyspace: &vindexes.Keyspace{Name: "ks_unmanaged", Sharded: true}, - ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(parentTbl, []string{"cola"}, []string{"cola"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"cola1", "cola2"}, []string{"ccola1", "ccola2"}, sqlparser.SetNull), - }, - }, - }, - "t2": &RealTable{ - Table: &vindexes.Table{ - Name: sqlparser.NewIdentifierCS("t2"), - Keyspace: &vindexes.Keyspace{Name: "ks"}, - }, - }, - "t3": &RealTable{ - Table: &vindexes.Table{ - Name: sqlparser.NewIdentifierCS("t3"), - Keyspace: &vindexes.Keyspace{Name: "undefined_ks", Sharded: true}, - }, - }, - "t4": &RealTable{ - Table: &vindexes.Table{ - Name: sqlparser.NewIdentifierCS("t4"), - Keyspace: &vindexes.Keyspace{Name: "ks"}, - ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), - ckInfo(parentTbl, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), - ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - }, - ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(parentTbl, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), - pkInfo(parentTbl, []string{"pcolc"}, []string{"colc"}), - pkInfo(parentTbl, []string{"pcolb", "pcola"}, []string{"colb", "cola"}), - pkInfo(parentTbl, []string{"pcolb"}, []string{"colb"}), - pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), - pkInfo(parentTbl, []string{"pcolb", "pcolx"}, []string{"colb", "colx"}), - }, - }, - }, - "t5": &RealTable{ - Table: &vindexes.Table{ - Name: sqlparser.NewIdentifierCS("t5"), - Keyspace: &vindexes.Keyspace{Name: "ks"}, - ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), - ckInfo(parentTbl, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), - }, - ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(parentTbl, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), - pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), - pkInfo(parentTbl, []string{"pcold", "pcolc"}, []string{"cold", "colc"}), - pkInfo(parentTbl, []string{"pcold"}, []string{"cold"}), - pkInfo(parentTbl, []string{"pcold", "pcolx"}, []string{"cold", "colx"}), - }, - }, - }, - "t6": &RealTable{ - Table: &vindexes.Table{ - Name: sqlparser.NewIdentifierCS("t6"), - Keyspace: &vindexes.Keyspace{Name: "ks"}, - ChildForeignKeys: []vindexes.ChildFKInfo{ - ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), - ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), - ckInfo(parentTbl, []string{"colx", "coly"}, []string{"child_colx", "child_coly"}, sqlparser.Cascade), - ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - }, - ParentForeignKeys: []vindexes.ParentFKInfo{ - pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), - pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), - }, - }, - }, -} - -// TestGetAllManagedForeignKeys tests the functionality of getAllManagedForeignKeys. -func TestGetAllManagedForeignKeys(t *testing.T) { - tests := []struct { - name string - analyzer *analyzer - childFkWanted map[TableSet][]vindexes.ChildFKInfo - parentFkWanted map[TableSet][]vindexes.ParentFKInfo - expectedErr string - }{ - { - name: "Collect all foreign key constraints", - analyzer: &analyzer{ - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t0"], - tbl["t1"], - &DerivedTable{}, - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - "ks_unmanaged": vschemapb.Keyspace_unmanaged, - }, - }, - }, - }, - childFkWanted: map[TableSet][]vindexes.ChildFKInfo{ - SingleTableSet(0): { - ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), - }, - }, - parentFkWanted: map[TableSet][]vindexes.ParentFKInfo{ - SingleTableSet(0): { - pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), - pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), - }, - }, - }, - { - name: "keyspace not found in schema information", - analyzer: &analyzer{ - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t2"], - tbl["t3"], - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - }, - }, - }, - }, - expectedErr: "undefined_ks keyspace not found", - }, - { - name: "Cyclic fk constraints error", - analyzer: &analyzer{ - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t0"], tbl["t1"], - &DerivedTable{}, - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - "ks_unmanaged": vschemapb.Keyspace_unmanaged, - }, - KsError: map[string]error{ - "ks": fmt.Errorf("VT09019: ks has cyclic foreign keys"), - }, - }, - }, - }, - expectedErr: "VT09019: ks has cyclic foreign keys", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - childFk, parentFk, err := tt.analyzer.getAllManagedForeignKeys() - if tt.expectedErr != "" { - require.EqualError(t, err, tt.expectedErr) - return - } - require.EqualValues(t, tt.childFkWanted, childFk) - require.EqualValues(t, tt.parentFkWanted, parentFk) - }) - } -} - -// TestFilterForeignKeysUsingUpdateExpressions tests the functionality of filterForeignKeysUsingUpdateExpressions. -func TestFilterForeignKeysUsingUpdateExpressions(t *testing.T) { - cola := sqlparser.NewColName("cola") - colb := sqlparser.NewColName("colb") - colc := sqlparser.NewColName("colc") - cold := sqlparser.NewColName("cold") - a := &analyzer{ - binder: &binder{ - direct: map[sqlparser.Expr]TableSet{ - cola: SingleTableSet(0), - colb: SingleTableSet(0), - colc: SingleTableSet(1), - cold: SingleTableSet(1), - }, - }, - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t4"], - tbl["t5"], - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - }, - }, - }, - } - updateExprs := sqlparser.UpdateExprs{ - &sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}, - &sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}, - &sqlparser.UpdateExpr{Name: colc, Expr: sqlparser.NewIntLiteral("1")}, - &sqlparser.UpdateExpr{Name: cold, Expr: &sqlparser.NullVal{}}, - } - tests := []struct { - name string - analyzer *analyzer - allChildFks map[TableSet][]vindexes.ChildFKInfo - allParentFks map[TableSet][]vindexes.ParentFKInfo - updExprs sqlparser.UpdateExprs - childFksWanted map[TableSet][]vindexes.ChildFKInfo - parentFksWanted map[TableSet][]vindexes.ParentFKInfo - }{ - { - name: "Child Foreign Keys Filtering", - analyzer: a, - allParentFks: nil, - allChildFks: map[TableSet][]vindexes.ChildFKInfo{ - SingleTableSet(0): tbl["t4"].(*RealTable).Table.ChildForeignKeys, - SingleTableSet(1): tbl["t5"].(*RealTable).Table.ChildForeignKeys, - }, - updExprs: updateExprs, - childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ - SingleTableSet(0): { - ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), - }, - SingleTableSet(1): { - ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), - }, - }, - parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{}, - }, { - name: "Parent Foreign Keys Filtering", - analyzer: a, - allParentFks: map[TableSet][]vindexes.ParentFKInfo{ - SingleTableSet(0): tbl["t4"].(*RealTable).Table.ParentForeignKeys, - SingleTableSet(1): tbl["t5"].(*RealTable).Table.ParentForeignKeys, - }, - allChildFks: nil, - updExprs: updateExprs, - childFksWanted: map[TableSet][]vindexes.ChildFKInfo{}, - parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ - SingleTableSet(0): { - pkInfo(parentTbl, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), - pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), - }, - SingleTableSet(1): { - pkInfo(parentTbl, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - childFks, parentFks, _ := tt.analyzer.filterForeignKeysUsingUpdateExpressions(tt.allChildFks, tt.allParentFks, tt.updExprs) - require.EqualValues(t, tt.childFksWanted, childFks) - require.EqualValues(t, tt.parentFksWanted, parentFks) - }) - } -} - -// TestGetInvolvedForeignKeys tests the functionality of getInvolvedForeignKeys. -func TestGetInvolvedForeignKeys(t *testing.T) { - cola := sqlparser.NewColName("cola") - colb := sqlparser.NewColName("colb") - colc := sqlparser.NewColName("colc") - cold := sqlparser.NewColName("cold") - tests := []struct { - name string - stmt sqlparser.Statement - analyzer *analyzer - childFksWanted map[TableSet][]vindexes.ChildFKInfo - parentFksWanted map[TableSet][]vindexes.ParentFKInfo - childFkUpdateExprsWanted map[string]sqlparser.UpdateExprs - expectedErr string - }{ - { - name: "Delete Query", - stmt: &sqlparser.Delete{}, - analyzer: &analyzer{ - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t0"], - tbl["t1"], - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - "ks_unmanaged": vschemapb.Keyspace_unmanaged, - }, - }, - }, - }, - childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ - SingleTableSet(0): { - ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), - }, - }, - }, - { - name: "Update statement", - stmt: &sqlparser.Update{ - Exprs: sqlparser.UpdateExprs{ - &sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}, - &sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}, - &sqlparser.UpdateExpr{Name: colc, Expr: sqlparser.NewIntLiteral("1")}, - &sqlparser.UpdateExpr{Name: cold, Expr: &sqlparser.NullVal{}}, - }, - }, - analyzer: &analyzer{ - binder: &binder{ - direct: map[sqlparser.Expr]TableSet{ - cola: SingleTableSet(0), - colb: SingleTableSet(0), - colc: SingleTableSet(1), - cold: SingleTableSet(1), - }, - }, - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t4"], - tbl["t5"], - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - }, - }, - }, - }, - childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ - SingleTableSet(0): { - ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), - }, - SingleTableSet(1): { - ckInfo(parentTbl, []string{"cold"}, []string{"child_cold"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"colc", "colx"}, []string{"child_colc", "child_colx"}, sqlparser.SetNull), - }, - }, - parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ - SingleTableSet(0): { - pkInfo(parentTbl, []string{"pcola", "pcolx"}, []string{"cola", "colx"}), - pkInfo(parentTbl, []string{"pcola"}, []string{"cola"}), - }, - SingleTableSet(1): { - pkInfo(parentTbl, []string{"pcolc", "pcolx"}, []string{"colc", "colx"}), - }, - }, - childFkUpdateExprsWanted: map[string]sqlparser.UpdateExprs{ - "ks.parentt|child_cola|child_colx||ks.t4|cola|colx": {&sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}}, - "ks.parentt|child_colb||ks.t4|colb": {&sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}}, - "ks.parentt|child_colc|child_colx||ks.t5|colc|colx": {&sqlparser.UpdateExpr{Name: colc, Expr: sqlparser.NewIntLiteral("1")}}, - "ks.parentt|child_cold||ks.t5|cold": {&sqlparser.UpdateExpr{Name: cold, Expr: &sqlparser.NullVal{}}}, - }, - }, - { - name: "Replace Query", - stmt: &sqlparser.Insert{ - Action: sqlparser.ReplaceAct, - }, - analyzer: &analyzer{ - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t0"], - tbl["t1"], - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - "ks_unmanaged": vschemapb.Keyspace_unmanaged, - }, - }, - }, - }, - childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ - SingleTableSet(0): { - ckInfo(parentTbl, []string{"col"}, []string{"col"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"col1", "col2"}, []string{"ccol1", "ccol2"}, sqlparser.SetNull), - }, - }, - parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ - SingleTableSet(0): { - pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), - pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), - }, - }, - }, - { - name: "Insert Query", - stmt: &sqlparser.Insert{ - Action: sqlparser.InsertAct, - }, - analyzer: &analyzer{ - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t0"], - tbl["t1"], - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - "ks_unmanaged": vschemapb.Keyspace_unmanaged, - }, - }, - }, - }, - childFksWanted: nil, - parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ - SingleTableSet(0): { - pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), - pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), - }, - }, - }, - { - name: "Insert Query with On Duplicate", - stmt: &sqlparser.Insert{ - Action: sqlparser.InsertAct, - OnDup: sqlparser.OnDup{ - &sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}, - &sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}, - }, - }, - analyzer: &analyzer{ - binder: &binder{ - direct: map[sqlparser.Expr]TableSet{ - cola: SingleTableSet(0), - colb: SingleTableSet(0), - }, - }, - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t6"], - tbl["t1"], - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - "ks_unmanaged": vschemapb.Keyspace_unmanaged, - }, - }, - }, - }, - childFksWanted: map[TableSet][]vindexes.ChildFKInfo{ - SingleTableSet(0): { - ckInfo(parentTbl, []string{"colb"}, []string{"child_colb"}, sqlparser.Restrict), - ckInfo(parentTbl, []string{"cola", "colx"}, []string{"child_cola", "child_colx"}, sqlparser.SetNull), - }, - }, - parentFksWanted: map[TableSet][]vindexes.ParentFKInfo{ - SingleTableSet(0): { - pkInfo(parentTbl, []string{"colb"}, []string{"colb"}), - pkInfo(parentTbl, []string{"colb1", "colb2"}, []string{"ccolb1", "ccolb2"}), - }, - }, - childFkUpdateExprsWanted: map[string]sqlparser.UpdateExprs{ - "ks.parentt|child_cola|child_colx||ks.t6|cola|colx": {&sqlparser.UpdateExpr{Name: cola, Expr: sqlparser.NewIntLiteral("1")}}, - "ks.parentt|child_colb||ks.t6|colb": {&sqlparser.UpdateExpr{Name: colb, Expr: &sqlparser.NullVal{}}}, - }, - }, - { - name: "Insert error", - stmt: &sqlparser.Insert{}, - analyzer: &analyzer{ - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t2"], - tbl["t3"], - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - }, - }, - }, - }, - expectedErr: "undefined_ks keyspace not found", - }, - { - name: "Update error", - stmt: &sqlparser.Update{}, - analyzer: &analyzer{ - tables: &tableCollector{ - Tables: []TableInfo{ - tbl["t2"], - tbl["t3"], - }, - si: &FakeSI{ - KsForeignKeyMode: map[string]vschemapb.Keyspace_ForeignKeyMode{ - "ks": vschemapb.Keyspace_managed, - }, - }, - }, - }, - expectedErr: "undefined_ks keyspace not found", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - childFks, parentFks, childFkUpdateExprs, err := tt.analyzer.getInvolvedForeignKeys(tt.stmt) - if tt.expectedErr != "" { - require.EqualError(t, err, tt.expectedErr) - return - } - require.EqualValues(t, tt.childFksWanted, childFks) - require.EqualValues(t, tt.childFkUpdateExprsWanted, childFkUpdateExprs) - require.EqualValues(t, tt.parentFksWanted, parentFks) - }) - } -} - -func ckInfo(cTable *vindexes.Table, pCols []string, cCols []string, refAction sqlparser.ReferenceAction) vindexes.ChildFKInfo { - return vindexes.ChildFKInfo{ - Table: cTable, - ParentColumns: sqlparser.MakeColumns(pCols...), - ChildColumns: sqlparser.MakeColumns(cCols...), - OnDelete: refAction, - } -} - -func pkInfo(parentTable *vindexes.Table, pCols []string, cCols []string) vindexes.ParentFKInfo { - return vindexes.ParentFKInfo{ - Table: parentTable, - ParentColumns: sqlparser.MakeColumns(pCols...), - ChildColumns: sqlparser.MakeColumns(cCols...), - } -}