From fa9e5942e1026df95e05353b3bb7d7245e7109b2 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 25 Apr 2024 14:20:14 +0200 Subject: [PATCH 1/8] Add unit test cases for reference table Signed-off-by: Rohit Nayak --- .../testdata/reference_table_cases.json | 320 ++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 go/vt/vtgate/planbuilder/testdata/reference_table_cases.json diff --git a/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json new file mode 100644 index 00000000000..b86b41e9ffa --- /dev/null +++ b/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json @@ -0,0 +1,320 @@ +[ + { + "comment": "delete from reference table with another name - query send to source table", + "query": "delete from user.ref_with_source where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table with another name - query send to source table", + "query": "update user.ref_with_source set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table with another name - query send to source table", + "query": "insert into user.ref_with_source(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into user.ref_with_source(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table - query send to source table", + "query": "delete from source_of_ref where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from source_of_ref where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table - query send to source table", + "query": "update source_of_ref set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update source_of_ref set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table - query send to source table", + "query": "insert into source_of_ref(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into source_of_ref(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table qualified with unsharded - query send to source table", + "query": "delete from main.source_of_ref where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from main.source_of_ref where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table qualified with unsharded - query send to source table", + "query": "update main.source_of_ref set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update main.source_of_ref set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table qualified with unsharded - query send to source table", + "query": "insert into main.source_of_ref(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into main.source_of_ref(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table with another name - query send to source table", + "query": "delete from user.ref_with_source where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table with another name - query send to source table", + "query": "update user.ref_with_source set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table with another name - query send to source table", + "query": "insert into user.ref_with_source(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into user.ref_with_source(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "select with join to reference table in sharded keyspace: should route shard-scoped", + "query": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "plan": { + "QueryType": "SELECT", + "Original": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "Instructions": { + "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", + "OperatorType": "Route", + "Variant": "EqualUnique", + "Vindex": "user_index", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", + "Table": "`user`, ref_with_source", + "Values": [ + "INT64(2)" + ] + }, + "TablesUsed": [ + "user.ref_with_source", + "user.user" + ] + } + }, + { + "comment": "select with join to reference table in unsharded keyspace: should route shard-scoped", + "query": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "plan": { + "QueryType": "SELECT", + "Original": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "Instructions": { + "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", + "OperatorType": "Route", + "Variant": "EqualUnique", + "Vindex": "user_index", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", + "Table": "`user`, ref_with_source", + "Values": [ + "INT64(2)" + ] + }, + "TablesUsed": [ + "user.ref_with_source", + "user.user" + ] + } + } +] From b29862dc9bb094d5895b79966666c0f4f153bb76 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 25 Apr 2024 15:50:47 +0200 Subject: [PATCH 2/8] Change expectation for bindvars Signed-off-by: Rohit Nayak --- go/vt/vtgate/planbuilder/testdata/reference_table_cases.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json index b86b41e9ffa..9be5e7ffc24 100644 --- a/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json @@ -281,7 +281,7 @@ "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", "Table": "`user`, ref_with_source", "Values": [ - "INT64(2)" + "2" ] }, "TablesUsed": [ @@ -308,7 +308,7 @@ "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", "Table": "`user`, ref_with_source", "Values": [ - "INT64(2)" + "2" ] }, "TablesUsed": [ From 5e23ca99df7b9a04018eed2237f05c5c7eaf0bb1 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 25 Apr 2024 15:52:07 +0200 Subject: [PATCH 3/8] Add unit test for reference table cases Signed-off-by: Rohit Nayak --- go/vt/vtgate/planbuilder/plan_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 5ac3cf8a7cc..47ec0937a43 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -837,3 +837,17 @@ func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCase } } } + +func TestReferenceRerouting(t *testing.T) { + reset := operators.EnableDebugPrinting() + defer reset() + + lv := loadSchema(t, "vschemas/schema.json", true) + vschema := &vschemawrapper.VSchemaWrapper{ + V: lv, + TestBuilder: TestBuilder, + Env: vtenv.NewTestEnv(), + } + + testFile(t, "reference_table_cases.json", "", vschema, false) +} From 2016aee0f53942af8e765ee9a5a1ff51d19e932b Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 25 Apr 2024 16:06:40 +0200 Subject: [PATCH 4/8] Make modifications to insert for rewriting reference tables. There is already logic for updates/deletes Signed-off-by: Rohit Nayak --- go/vt/vtgate/planbuilder/operators/insert.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/go/vt/vtgate/planbuilder/operators/insert.go b/go/vt/vtgate/planbuilder/operators/insert.go index 9d0048d9322..77036425a2a 100644 --- a/go/vt/vtgate/planbuilder/operators/insert.go +++ b/go/vt/vtgate/planbuilder/operators/insert.go @@ -109,6 +109,16 @@ func createOperatorFromInsert(ctx *plancontext.PlanningContext, ins *sqlparser.I vTbl, routing := buildVindexTableForDML(ctx, tableInfo, qt, "insert") + if tableInfo.GetVindexTable().Type == vindexes.TypeReference { + // If the table being inserted into is a reference table, we need to insert into the source table. + tname, _ := tableInfo.Name() + if vTbl.Name != tname.Name || vTbl.Keyspace.Name == tname.Name.String() { // optimization + newAliasTbl := sqlparser.NewAliasedTableExpr(vTbl.GetTableName(), "") + ins.Table.Expr = newAliasTbl.Expr + ins.Table.As = newAliasTbl.As + } + } + deleteBeforeInsert := false if ins.Action == sqlparser.ReplaceAct && (ctx.SemTable.ForeignKeysPresent() || vTbl.Keyspace.Sharded) && From 745463b305e2c4900579f166c9f8b736fc3922e1 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Thu, 25 Apr 2024 16:07:07 +0200 Subject: [PATCH 5/8] Add e2e test for reference tables Signed-off-by: Rohit Nayak --- .../endtoend/vreplication/reference_test.go | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 go/test/endtoend/vreplication/reference_test.go diff --git a/go/test/endtoend/vreplication/reference_test.go b/go/test/endtoend/vreplication/reference_test.go new file mode 100644 index 00000000000..f29db50f29e --- /dev/null +++ b/go/test/endtoend/vreplication/reference_test.go @@ -0,0 +1,150 @@ +package vreplication + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + uksSchema = ` +create table product (id int, mfg_id int, cat_id int, name varchar(128), primary key(id)); +create table cat (id int, name varchar(128), primary key(id)); +create table mfg (id int, name varchar(128), primary key(id)); +` + sksSchema = ` +create table product (id int, mfg_id int, cat_id int, name varchar(128), primary key(id)); +create table cat (id int, name varchar(128), primary key(id)); +create table mfg2 (id int, name varchar(128), primary key(id)); +` + uksVSchema = ` +{ + "sharded": false, + "tables": { + "product": {}, + "cat": {}, + "mfg": {} + } +}` + + sksVSchema = ` +{ + "sharded": true, + "tables": { + "product": { + "column_vindexes": [ + { + "column": "id", + "name": "hash" + } + ] + }, + "cat": { + "type": "reference", + "source": "uks.cat" + }, + "mfg2": { + "type": "reference", + "source": "uks.mfg" + } + }, + "vindexes": { + "hash": { + "type": "hash" + } + } +}` + materializeCatSpec = ` +{ + "workflow": "wfCat", + "source_keyspace": "uks", + "target_keyspace": "sks", + "table_settings": [ {"target_table": "cat", "source_expression": "select id, name from cat" }] +}` + materializeMfgSpec = ` +{ + "workflow": "wfMfg", + "source_keyspace": "uks", + "target_keyspace": "sks", + "table_settings": [ {"target_table": "mfg2", "source_expression": "select id, name from mfg" }] +}` + initializeTables = ` +use uks; +insert into product values (1, 1, 1, 'p1'); +insert into product values (2, 2, 2, 'p2'); +insert into product values (3, 3, 3, 'p3'); +insert into cat values (1, 'c1'); +insert into cat values (2, 'c2'); +insert into cat values (3, 'c3'); +insert into mfg values (1, 'm1'); +insert into mfg values (2, 'm2'); +insert into mfg values (3, 'm3'); +insert into mfg values (4, 'm4'); +` +) + +func TestReferenceTableMaterializationAndRouting(t *testing.T) { + var err error + defaultCellName := "zone1" + vc = NewVitessCluster(t, nil) + defer vc.TearDown() + defaultReplicas = 0 // because of CI resource constraints we can only run this test with primary tablets + defer func() { defaultReplicas = 1 }() + uks := "uks" + sks := "sks" + + defaultCell := vc.Cells[defaultCellName] + vc.AddKeyspace(t, []*Cell{defaultCell}, uks, "0", uksVSchema, uksSchema, defaultReplicas, defaultRdonly, 100, nil) + vc.AddKeyspace(t, []*Cell{defaultCell}, sks, "-80,80-", sksVSchema, sksSchema, defaultReplicas, defaultRdonly, 200, nil) + vtgateConn := getConnection(t, vc.ClusterConfig.hostname, vc.ClusterConfig.vtgateMySQLPort) + + verifyClusterHealth(t, vc) + _, _, err = vtgateConn.ExecuteFetchMulti(initializeTables, 0, false) + require.NoError(t, err) + vtgateConn.Close() + + materialize(t, materializeCatSpec, false) + materialize(t, materializeMfgSpec, false) + + tabDash80 := vc.getPrimaryTablet(t, sks, "-80") + tab80Dash := vc.getPrimaryTablet(t, sks, "80-") + catchup(t, tabDash80, "wfCat", "Materialize Category") + catchup(t, tab80Dash, "wfCat", "Materialize Category") + catchup(t, tabDash80, "wfMfg", "Materialize Manufacturer") + catchup(t, tab80Dash, "wfMfg", "Materialize Manufacturer") + + vtgateConn = getConnection(t, vc.ClusterConfig.hostname, vc.ClusterConfig.vtgateMySQLPort) + defer vtgateConn.Close() + waitForRowCount(t, vtgateConn, sks, "cat", 3) + waitForRowCount(t, vtgateConn, sks, "mfg2", 4) + + execRefQuery(t, "insert into mfg values (5, 'm5')") + execRefQuery(t, "insert into mfg2 values (6, 'm6')") + execRefQuery(t, "insert into uks.mfg values (7, 'm7')") + execRefQuery(t, "insert into sks.mfg2 values (8, 'm8')") + waitForRowCount(t, vtgateConn, uks, "mfg", 8) + + execRefQuery(t, "update mfg set name = concat(name, '-updated') where id = 1") + execRefQuery(t, "update mfg2 set name = concat(name, '-updated') where id = 2") + execRefQuery(t, "update uks.mfg set name = concat(name, '-updated') where id = 3") + execRefQuery(t, "update sks.mfg2 set name = concat(name, '-updated') where id = 4") + + waitForRowCount(t, vtgateConn, uks, "mfg", 8) + qr := execVtgateQuery(t, vtgateConn, "uks", "select count(*) from uks.mfg where name like '%updated%'") + require.NotNil(t, qr) + require.Equal(t, "4", qr.Rows[0][0].ToString()) + + execRefQuery(t, "delete from mfg where id = 5") + execRefQuery(t, "delete from mfg2 where id = 6") + execRefQuery(t, "delete from uks.mfg where id = 7") + execRefQuery(t, "delete from sks.mfg2 where id = 8") + waitForRowCount(t, vtgateConn, uks, "mfg", 4) + +} + +func execRefQuery(t *testing.T, query string) { + vtgateConn := getConnection(t, vc.ClusterConfig.hostname, vc.ClusterConfig.vtgateMySQLPort) + defer vtgateConn.Close() + _, err := vtgateConn.ExecuteFetch(query, 0, false) + require.NoError(t, err) +} From 0eb543ad64f5b2a7bc3643c5a6faf64e8fa9c2e8 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Fri, 26 Apr 2024 17:24:46 +0200 Subject: [PATCH 6/8] Address review comments. Merge newly added tests into existing ones Signed-off-by: Rohit Nayak --- go/vt/vtgate/planbuilder/operators/insert.go | 2 +- go/vt/vtgate/planbuilder/plan_test.go | 14 - .../planbuilder/testdata/reference_cases.json | 318 +++++++++++++++++ .../testdata/reference_table_cases.json | 320 ------------------ 4 files changed, 319 insertions(+), 335 deletions(-) delete mode 100644 go/vt/vtgate/planbuilder/testdata/reference_table_cases.json diff --git a/go/vt/vtgate/planbuilder/operators/insert.go b/go/vt/vtgate/planbuilder/operators/insert.go index 77036425a2a..a818aa9d0a7 100644 --- a/go/vt/vtgate/planbuilder/operators/insert.go +++ b/go/vt/vtgate/planbuilder/operators/insert.go @@ -109,7 +109,7 @@ func createOperatorFromInsert(ctx *plancontext.PlanningContext, ins *sqlparser.I vTbl, routing := buildVindexTableForDML(ctx, tableInfo, qt, "insert") - if tableInfo.GetVindexTable().Type == vindexes.TypeReference { + if tableInfo.GetVindexTable().Type == vindexes.TypeReference && tableInfo.GetVindexTable().Source != nil { // If the table being inserted into is a reference table, we need to insert into the source table. tname, _ := tableInfo.Name() if vTbl.Name != tname.Name || vTbl.Keyspace.Name == tname.Name.String() { // optimization diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 47ec0937a43..5ac3cf8a7cc 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -837,17 +837,3 @@ func benchmarkPlanner(b *testing.B, version plancontext.PlannerVersion, testCase } } } - -func TestReferenceRerouting(t *testing.T) { - reset := operators.EnableDebugPrinting() - defer reset() - - lv := loadSchema(t, "vschemas/schema.json", true) - vschema := &vschemawrapper.VSchemaWrapper{ - V: lv, - TestBuilder: TestBuilder, - Env: vtenv.NewTestEnv(), - } - - testFile(t, "reference_table_cases.json", "", vschema, false) -} diff --git a/go/vt/vtgate/planbuilder/testdata/reference_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_cases.json index 91aac12d9e9..a89fa103923 100644 --- a/go/vt/vtgate/planbuilder/testdata/reference_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/reference_cases.json @@ -428,5 +428,323 @@ "main.global_ref" ] } + }, + { + "comment": "delete from reference table with another name - query send to source table", + "query": "delete from user.ref_with_source where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table with another name - query send to source table", + "query": "update user.ref_with_source set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table with another name - query send to source table", + "query": "insert into user.ref_with_source(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into user.ref_with_source(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table - query send to source table", + "query": "delete from source_of_ref where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from source_of_ref where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table - query send to source table", + "query": "update source_of_ref set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update source_of_ref set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table - query send to source table", + "query": "insert into source_of_ref(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into source_of_ref(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table qualified with unsharded - query send to source table", + "query": "delete from main.source_of_ref where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from main.source_of_ref where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table qualified with unsharded - query send to source table", + "query": "update main.source_of_ref set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update main.source_of_ref set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table qualified with unsharded - query send to source table", + "query": "insert into main.source_of_ref(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into main.source_of_ref(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "delete from reference table with another name - query send to source table", + "query": "delete from user.ref_with_source where col = 1", + "plan": { + "QueryType": "DELETE", + "Original": "delete from user.ref_with_source where col = 1", + "Instructions": { + "OperatorType": "Delete", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "delete from source_of_ref where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "update from reference table with another name - query send to source table", + "query": "update user.ref_with_source set x = 4 where col = 1", + "plan": { + "QueryType": "UPDATE", + "Original": "update user.ref_with_source set x = 4 where col = 1", + "Instructions": { + "OperatorType": "Update", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "update source_of_ref set x = 4 where col = 1", + "Table": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "insert from reference table with another name - query send to source table", + "query": "insert into user.ref_with_source(x) values(4)", + "plan": { + "QueryType": "INSERT", + "Original": "insert into user.ref_with_source(x) values(4)", + "Instructions": { + "OperatorType": "Insert", + "Variant": "Unsharded", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "TargetTabletType": "PRIMARY", + "Query": "insert into source_of_ref(x) values (4)", + "TableName": "source_of_ref" + }, + "TablesUsed": [ + "main.source_of_ref" + ] + } + }, + { + "comment": "select with join to reference table in sharded keyspace: should route shard-scoped", + "query": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "plan": { + "QueryType": "SELECT", + "Original": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "Instructions": { + "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", + "OperatorType": "Route", + "Variant": "EqualUnique", + "Vindex": "user_index", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", + "Table": "`user`, ref_with_source", + "Values": [ + "2" + ] + }, + "TablesUsed": [ + "user.ref_with_source", + "user.user" + ] + } + }, + { + "comment": "select with join to reference table in unsharded keyspace: should route shard-scoped", + "query": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "plan": { + "QueryType": "SELECT", + "Original": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", + "Instructions": { + "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", + "OperatorType": "Route", + "Variant": "EqualUnique", + "Vindex": "user_index", + "Keyspace": { + "Name": "user", + "Sharded": true + }, + "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", + "Table": "`user`, ref_with_source", + "Values": [ + "2" + ] + }, + "TablesUsed": [ + "user.ref_with_source", + "user.user" + ] + } } ] diff --git a/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json b/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json deleted file mode 100644 index 9be5e7ffc24..00000000000 --- a/go/vt/vtgate/planbuilder/testdata/reference_table_cases.json +++ /dev/null @@ -1,320 +0,0 @@ -[ - { - "comment": "delete from reference table with another name - query send to source table", - "query": "delete from user.ref_with_source where col = 1", - "plan": { - "QueryType": "DELETE", - "Original": "delete from user.ref_with_source where col = 1", - "Instructions": { - "OperatorType": "Delete", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "update from reference table with another name - query send to source table", - "query": "update user.ref_with_source set x = 4 where col = 1", - "plan": { - "QueryType": "UPDATE", - "Original": "update user.ref_with_source set x = 4 where col = 1", - "Instructions": { - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "insert from reference table with another name - query send to source table", - "query": "insert into user.ref_with_source(x) values(4)", - "plan": { - "QueryType": "INSERT", - "Original": "insert into user.ref_with_source(x) values(4)", - "Instructions": { - "OperatorType": "Insert", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "insert into source_of_ref(x) values (4)", - "TableName": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "delete from reference table - query send to source table", - "query": "delete from source_of_ref where col = 1", - "plan": { - "QueryType": "DELETE", - "Original": "delete from source_of_ref where col = 1", - "Instructions": { - "OperatorType": "Delete", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "update from reference table - query send to source table", - "query": "update source_of_ref set x = 4 where col = 1", - "plan": { - "QueryType": "UPDATE", - "Original": "update source_of_ref set x = 4 where col = 1", - "Instructions": { - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "insert from reference table - query send to source table", - "query": "insert into source_of_ref(x) values(4)", - "plan": { - "QueryType": "INSERT", - "Original": "insert into source_of_ref(x) values(4)", - "Instructions": { - "OperatorType": "Insert", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "insert into source_of_ref(x) values (4)", - "TableName": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "delete from reference table qualified with unsharded - query send to source table", - "query": "delete from main.source_of_ref where col = 1", - "plan": { - "QueryType": "DELETE", - "Original": "delete from main.source_of_ref where col = 1", - "Instructions": { - "OperatorType": "Delete", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "update from reference table qualified with unsharded - query send to source table", - "query": "update main.source_of_ref set x = 4 where col = 1", - "plan": { - "QueryType": "UPDATE", - "Original": "update main.source_of_ref set x = 4 where col = 1", - "Instructions": { - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "insert from reference table qualified with unsharded - query send to source table", - "query": "insert into main.source_of_ref(x) values(4)", - "plan": { - "QueryType": "INSERT", - "Original": "insert into main.source_of_ref(x) values(4)", - "Instructions": { - "OperatorType": "Insert", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "insert into source_of_ref(x) values (4)", - "TableName": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "delete from reference table with another name - query send to source table", - "query": "delete from user.ref_with_source where col = 1", - "plan": { - "QueryType": "DELETE", - "Original": "delete from user.ref_with_source where col = 1", - "Instructions": { - "OperatorType": "Delete", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "delete from source_of_ref where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "update from reference table with another name - query send to source table", - "query": "update user.ref_with_source set x = 4 where col = 1", - "plan": { - "QueryType": "UPDATE", - "Original": "update user.ref_with_source set x = 4 where col = 1", - "Instructions": { - "OperatorType": "Update", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "update source_of_ref set x = 4 where col = 1", - "Table": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "insert from reference table with another name - query send to source table", - "query": "insert into user.ref_with_source(x) values(4)", - "plan": { - "QueryType": "INSERT", - "Original": "insert into user.ref_with_source(x) values(4)", - "Instructions": { - "OperatorType": "Insert", - "Variant": "Unsharded", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetTabletType": "PRIMARY", - "Query": "insert into source_of_ref(x) values (4)", - "TableName": "source_of_ref" - }, - "TablesUsed": [ - "main.source_of_ref" - ] - } - }, - { - "comment": "select with join to reference table in sharded keyspace: should route shard-scoped", - "query": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", - "plan": { - "QueryType": "SELECT", - "Original": "select * from user.ref_with_source ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", - "Instructions": { - "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", - "OperatorType": "Route", - "Variant": "EqualUnique", - "Vindex": "user_index", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", - "Table": "`user`, ref_with_source", - "Values": [ - "2" - ] - }, - "TablesUsed": [ - "user.ref_with_source", - "user.user" - ] - } - }, - { - "comment": "select with join to reference table in unsharded keyspace: should route shard-scoped", - "query": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", - "plan": { - "QueryType": "SELECT", - "Original": "select * from source_of_ref ref, `user`.`user` u where ref.id = u.ref_id and u.id = 2", - "Instructions": { - "FieldQuery": "select * from ref_with_source as ref, `user` as u where 1 != 1", - "OperatorType": "Route", - "Variant": "EqualUnique", - "Vindex": "user_index", - "Keyspace": { - "Name": "user", - "Sharded": true - }, - "Query": "select * from ref_with_source as ref, `user` as u where u.id = 2 and ref.id = u.ref_id", - "Table": "`user`, ref_with_source", - "Values": [ - "2" - ] - }, - "TablesUsed": [ - "user.ref_with_source", - "user.user" - ] - } - } -] From 18ef23e9a2e64e8e938b70031c8634091ec398df Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Wed, 1 May 2024 21:00:31 +0200 Subject: [PATCH 7/8] Fix logic for modifying the target being inserted into Signed-off-by: Rohit Nayak --- go/vt/vtgate/planbuilder/operators/insert.go | 12 +----------- go/vt/vtgate/planbuilder/operators/route_planning.go | 6 +++++- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operators/insert.go b/go/vt/vtgate/planbuilder/operators/insert.go index a818aa9d0a7..7c6e242ae9c 100644 --- a/go/vt/vtgate/planbuilder/operators/insert.go +++ b/go/vt/vtgate/planbuilder/operators/insert.go @@ -107,17 +107,7 @@ func (i *Insert) Statement() sqlparser.Statement { func createOperatorFromInsert(ctx *plancontext.PlanningContext, ins *sqlparser.Insert) Operator { tableInfo, qt := createQueryTableForDML(ctx, ins.Table, nil) - vTbl, routing := buildVindexTableForDML(ctx, tableInfo, qt, "insert") - - if tableInfo.GetVindexTable().Type == vindexes.TypeReference && tableInfo.GetVindexTable().Source != nil { - // If the table being inserted into is a reference table, we need to insert into the source table. - tname, _ := tableInfo.Name() - if vTbl.Name != tname.Name || vTbl.Keyspace.Name == tname.Name.String() { // optimization - newAliasTbl := sqlparser.NewAliasedTableExpr(vTbl.GetTableName(), "") - ins.Table.Expr = newAliasTbl.Expr - ins.Table.As = newAliasTbl.As - } - } + vTbl, routing := buildVindexTableForDML(ctx, tableInfo, qt, ins, "insert") deleteBeforeInsert := false if ins.Action == sqlparser.ReplaceAct && diff --git a/go/vt/vtgate/planbuilder/operators/route_planning.go b/go/vt/vtgate/planbuilder/operators/route_planning.go index f2cf3116f72..47405e3b935 100644 --- a/go/vt/vtgate/planbuilder/operators/route_planning.go +++ b/go/vt/vtgate/planbuilder/operators/route_planning.go @@ -80,15 +80,19 @@ func buildVindexTableForDML( ctx *plancontext.PlanningContext, tableInfo semantics.TableInfo, table *QueryTable, + ins *sqlparser.Insert, dmlType string, ) (*vindexes.Table, Routing) { vindexTable := tableInfo.GetVindexTable() - if vindexTable.Source != nil { + if tableInfo.GetVindexTable().Type == vindexes.TypeReference && vindexTable.Source != nil { sourceTable, _, _, _, _, err := ctx.VSchema.FindTableOrVindex(vindexTable.Source.TableName) if err != nil { panic(err) } vindexTable = sourceTable + refTbl := sqlparser.NewAliasedTableExpr(vindexTable.GetTableName(), "") + ins.Table.Expr = refTbl.Expr + // We don't need to process the alias because you cannot define aliases for inserts. } if !vindexTable.Keyspace.Sharded { From a7e2c2ffb88ab3ef193bc9f3ed6fb79f9b59f2b9 Mon Sep 17 00:00:00 2001 From: Rohit Nayak Date: Mon, 6 May 2024 16:11:09 +0200 Subject: [PATCH 8/8] Add license header Signed-off-by: Rohit Nayak --- go/test/endtoend/vreplication/reference_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/go/test/endtoend/vreplication/reference_test.go b/go/test/endtoend/vreplication/reference_test.go index f29db50f29e..8ff77de8708 100644 --- a/go/test/endtoend/vreplication/reference_test.go +++ b/go/test/endtoend/vreplication/reference_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package vreplication import (