From 66015fd318b93daeb70a76837dc3fea2e90559e1 Mon Sep 17 00:00:00 2001 From: Max Englander Date: Fri, 21 Oct 2022 18:01:53 -0400 Subject: [PATCH] optionally disable verify-after-insert behavior of lookup vindexes (#11313) Add a VIndex option to skip verify-after-insert behavior when using INSERT...IGNORE or INSERT...ON DUPLICATE KEY UPDATE. Co-authored-by: Harshit Gangal Co-authored-by: Jacques Grove Signed-off-by: Max Englander Signed-off-by: Max Englander --- .../vtgate/vindex_bindvars/main_test.go | 121 +++++++++++------- go/vt/vtexplain/testdata/test-schema.sql | 6 + go/vt/vtexplain/testdata/test-vschema.json | 18 +++ go/vt/vtgate/executor_dml_test.go | 95 +++++++++----- go/vt/vtgate/executor_framework_test.go | 23 +++- go/vt/vtgate/executor_test.go | 6 +- go/vt/vtgate/vindexes/lookup.go | 17 ++- go/vt/vtgate/vindexes/lookup_hash.go | 2 +- go/vt/vtgate/vindexes/lookup_test.go | 46 +++++++ 9 files changed, 244 insertions(+), 90 deletions(-) diff --git a/go/test/endtoend/vtgate/vindex_bindvars/main_test.go b/go/test/endtoend/vtgate/vindex_bindvars/main_test.go index 7900d155932..195ea4a6862 100644 --- a/go/test/endtoend/vtgate/vindex_bindvars/main_test.go +++ b/go/test/endtoend/vtgate/vindex_bindvars/main_test.go @@ -41,6 +41,7 @@ var ( id BIGINT NOT NULL, field BIGINT NOT NULL, field2 BIGINT, + field3 BIGINT, PRIMARY KEY (id) ) ENGINE=Innodb; @@ -56,6 +57,12 @@ CREATE TABLE lookup2 ( UNIQUE KEY (field2) ) ENGINE=Innodb; +CREATE TABLE lookup3 ( + field3 BIGINT NOT NULL, + keyspace_id binary(8), + UNIQUE KEY (field3) +) ENGINE=Innodb; + CREATE TABLE thex ( id VARBINARY(64) NOT NULL, field BIGINT NOT NULL, @@ -88,7 +95,7 @@ CREATE TABLE thex ( "table": "lookup1", "from": "field", "to": "keyspace_id", - "ignore_nulls": "true" + "ignore_nulls": "true" }, "owner": "t1" }, @@ -98,7 +105,17 @@ CREATE TABLE thex ( "table": "lookup2", "from": "field2", "to": "keyspace_id", - "ignore_nulls": "true" + "ignore_nulls": "true" + }, + "owner": "t1" + }, + "lookup3": { + "type": "lookup", + "params": { + "from": "field3", + "no_verify": "true", + "table": "lookup3", + "to": "keyspace_id" }, "owner": "t1" } @@ -117,6 +134,10 @@ CREATE TABLE thex ( { "column": "field2", "name": "lookup2" + }, + { + "column": "field3", + "name": "lookup3" } ] }, @@ -136,6 +157,14 @@ CREATE TABLE thex ( } ] }, + "lookup3": { + "column_vindexes": [ + { + "column": "field3", + "name": "binary_md5_vdx" + } + ] + }, "thex": { "column_vindexes": [ { @@ -216,51 +245,51 @@ func TestVindexBindVarOverlap(t *testing.T) { require.Nil(t, err) defer conn.Close() - utils.Exec(t, conn, "INSERT INTO t1 (id, field, field2) VALUES "+ - "(0,1,2), "+ - "(1,2,3), "+ - "(2,3,4), "+ - "(3,4,5), "+ - "(4,5,6), "+ - "(5,6,7), "+ - "(6,7,8), "+ - "(7,8,9), "+ - "(8,9,10), "+ - "(9,10,11), "+ - "(10,11,12), "+ - "(11,12,13), "+ - "(12,13,14), "+ - "(13,14,15), "+ - "(14,15,16), "+ - "(15,16,17), "+ - "(16,17,18), "+ - "(17,18,19), "+ - "(18,19,20), "+ - "(19,20,21), "+ - "(20,21,22)") - result := utils.Exec(t, conn, "select id, field, field2 from t1 order by id") + utils.Exec(t, conn, "INSERT INTO t1 (id, field, field2, field3) VALUES "+ + "(0,1,2,3), "+ + "(1,2,3,4), "+ + "(2,3,4,5), "+ + "(3,4,5,6), "+ + "(4,5,6,7), "+ + "(5,6,7,8), "+ + "(6,7,8,9), "+ + "(7,8,9,10), "+ + "(8,9,10,11), "+ + "(9,10,11,12), "+ + "(10,11,12,13), "+ + "(11,12,13,14), "+ + "(12,13,14,15), "+ + "(13,14,15,16), "+ + "(14,15,16,17), "+ + "(15,16,17,18), "+ + "(16,17,18,19), "+ + "(17,18,19,20), "+ + "(18,19,20,21), "+ + "(19,20,21,22), "+ + "(20,21,22,23)") + result := utils.Exec(t, conn, "select id, field, field2, field3 from t1 order by id") expected := - "[[INT64(0) INT64(1) INT64(2)] " + - "[INT64(1) INT64(2) INT64(3)] " + - "[INT64(2) INT64(3) INT64(4)] " + - "[INT64(3) INT64(4) INT64(5)] " + - "[INT64(4) INT64(5) INT64(6)] " + - "[INT64(5) INT64(6) INT64(7)] " + - "[INT64(6) INT64(7) INT64(8)] " + - "[INT64(7) INT64(8) INT64(9)] " + - "[INT64(8) INT64(9) INT64(10)] " + - "[INT64(9) INT64(10) INT64(11)] " + - "[INT64(10) INT64(11) INT64(12)] " + - "[INT64(11) INT64(12) INT64(13)] " + - "[INT64(12) INT64(13) INT64(14)] " + - "[INT64(13) INT64(14) INT64(15)] " + - "[INT64(14) INT64(15) INT64(16)] " + - "[INT64(15) INT64(16) INT64(17)] " + - "[INT64(16) INT64(17) INT64(18)] " + - "[INT64(17) INT64(18) INT64(19)] " + - "[INT64(18) INT64(19) INT64(20)] " + - "[INT64(19) INT64(20) INT64(21)] " + - "[INT64(20) INT64(21) INT64(22)]]" + "[[INT64(0) INT64(1) INT64(2) INT64(3)] " + + "[INT64(1) INT64(2) INT64(3) INT64(4)] " + + "[INT64(2) INT64(3) INT64(4) INT64(5)] " + + "[INT64(3) INT64(4) INT64(5) INT64(6)] " + + "[INT64(4) INT64(5) INT64(6) INT64(7)] " + + "[INT64(5) INT64(6) INT64(7) INT64(8)] " + + "[INT64(6) INT64(7) INT64(8) INT64(9)] " + + "[INT64(7) INT64(8) INT64(9) INT64(10)] " + + "[INT64(8) INT64(9) INT64(10) INT64(11)] " + + "[INT64(9) INT64(10) INT64(11) INT64(12)] " + + "[INT64(10) INT64(11) INT64(12) INT64(13)] " + + "[INT64(11) INT64(12) INT64(13) INT64(14)] " + + "[INT64(12) INT64(13) INT64(14) INT64(15)] " + + "[INT64(13) INT64(14) INT64(15) INT64(16)] " + + "[INT64(14) INT64(15) INT64(16) INT64(17)] " + + "[INT64(15) INT64(16) INT64(17) INT64(18)] " + + "[INT64(16) INT64(17) INT64(18) INT64(19)] " + + "[INT64(17) INT64(18) INT64(19) INT64(20)] " + + "[INT64(18) INT64(19) INT64(20) INT64(21)] " + + "[INT64(19) INT64(20) INT64(21) INT64(22)] " + + "[INT64(20) INT64(21) INT64(22) INT64(23)]]" assert.Equal(t, expected, fmt.Sprintf("%v", result.Rows)) } diff --git a/go/vt/vtexplain/testdata/test-schema.sql b/go/vt/vtexplain/testdata/test-schema.sql index d5f9fbad56f..716f141e472 100644 --- a/go/vt/vtexplain/testdata/test-schema.sql +++ b/go/vt/vtexplain/testdata/test-schema.sql @@ -111,3 +111,9 @@ CREATE TABLE orders_id_lookup ( keyspace_id varbinary(128), primary key(id) ); + +CREATE TABLE orders_id_lookup_no_verify ( + id int NOT NULL, + keyspace_id varbinary(128), + primary key(id) +); diff --git a/go/vt/vtexplain/testdata/test-vschema.json b/go/vt/vtexplain/testdata/test-vschema.json index f4350efa56d..ec25beaec50 100644 --- a/go/vt/vtexplain/testdata/test-vschema.json +++ b/go/vt/vtexplain/testdata/test-vschema.json @@ -19,6 +19,16 @@ }, "owner": "orders" }, + "orders_id_vdx_no_verify": { + "type": "lookup_unique", + "params": { + "table": "orders_id_lookup_no_verify", + "from": "id", + "to": "keyspace_id", + "no_verify": "true" + }, + "owner": "orders" + }, "music_user_map": { "type": "lookup_hash_unique", "owner": "music", @@ -165,6 +175,14 @@ } ] }, + "orders_id_lookup_no_verify": { + "column_vindexes": [ + { + "column": "id", + "name": "hash" + } + ] + }, "email_customer_map": { "column_vindexes": [ { diff --git a/go/vt/vtgate/executor_dml_test.go b/go/vt/vtgate/executor_dml_test.go index 05fcb59255d..634684936c2 100644 --- a/go/vt/vtgate/executor_dml_test.go +++ b/go/vt/vtgate/executor_dml_test.go @@ -164,21 +164,21 @@ func TestUpdateFromSubQuery(t *testing.T) { testQueryLog(t, logChan, "TestExecute", "UPDATE", "update user set a=(select count(*) from user where id = 3) where id = 1", 2) } -func TestUpdateEqualWithWriteOnlyLookupUniqueVindex(t *testing.T) { +func TestUpdateEqualWithNoVerifyAndWriteOnlyLookupUniqueVindexes(t *testing.T) { res := []*sqltypes.Result{sqltypes.MakeTestResult( - sqltypes.MakeTestFields("id|wo_lu_col|lu_col|t2_lu_vdx", "int64|int64|int64|int64"), - "1|2|1|0", + sqltypes.MakeTestFields("id|wo_lu_col|nv_lu_col|lu_col|t2_lu_vdx", "int64|int64|int64|int64|int64"), + "1|2|2|1|0", )} executor, sbc1, sbc2, sbcLookup := createCustomExecutorSetValues(executorVSchema, res) - _, err := executorExec(executor, "update t2_wo_lookup set lu_col = 5 where wo_lu_col = 2", nil) + _, err := executorExec(executor, "update t2_lookup set lu_col = 5 where wo_lu_col = 2", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{ { - Sql: "select id, wo_lu_col, lu_col, lu_col = 5 from t2_wo_lookup where wo_lu_col = 2 for update", + Sql: "select id, wo_lu_col, nv_lu_col, lu_col, lu_col = 5 from t2_lookup where wo_lu_col = 2 for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "update t2_wo_lookup set lu_col = 5 where wo_lu_col = 2", + Sql: "update t2_lookup set lu_col = 5 where wo_lu_col = 2", BindVariables: map[string]*querypb.BindVariable{}, }} @@ -513,18 +513,18 @@ func TestUpdateEqualWithMultipleLookupVindex(t *testing.T) { )}) sbc1.SetResults([]*sqltypes.Result{sqltypes.MakeTestResult( - sqltypes.MakeTestFields("id|wo_lu_col|lu_col|t2_lu_vdx", "int64|int64|int64|int64"), - "1|2|1|0", + sqltypes.MakeTestFields("id|wo_lu_col|nv_lu_col|lu_col|t2_lu_vdx", "int64|int64|int64|int64|int64"), + "1|2|2|1|0", )}) - _, err := executorExec(executor, "update t2_wo_lookup set lu_col = 5 where wo_lu_col = 2 and lu_col = 1", nil) + _, err := executorExec(executor, "update t2_lookup set lu_col = 5 where wo_lu_col = 2 and lu_col = 1", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{ { - Sql: "select id, wo_lu_col, lu_col, lu_col = 5 from t2_wo_lookup where wo_lu_col = 2 and lu_col = 1 for update", + Sql: "select id, wo_lu_col, nv_lu_col, lu_col, lu_col = 5 from t2_lookup where wo_lu_col = 2 and lu_col = 1 for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "update t2_wo_lookup set lu_col = 5 where wo_lu_col = 2 and lu_col = 1", + Sql: "update t2_lookup set lu_col = 5 where wo_lu_col = 2 and lu_col = 1", BindVariables: map[string]*querypb.BindVariable{}, }} @@ -564,19 +564,19 @@ func TestUpdateUseHigherCostVindexIfBackfilling(t *testing.T) { )}) sbc1.SetResults([]*sqltypes.Result{sqltypes.MakeTestResult( - sqltypes.MakeTestFields("id|wo_lu_col|lu_col|t2_lu_vdx", "int64|int64|int64|int64"), - "1|2|1|0", - "1|2|2|0", + sqltypes.MakeTestFields("id|wo_lu_col|nv_lu_col|lu_col|t2_lu_vdx", "int64|int64|int64|int64|int64"), + "1|2|2|1|0", + "1|2|2|2|0", )}) - _, err := executorExec(executor, "update t2_wo_lookup set lu_col = 5 where wo_lu_col = 2 and lu_col in (1, 2)", nil) + _, err := executorExec(executor, "update t2_lookup set lu_col = 5 where wo_lu_col = 2 and lu_col in (1, 2)", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{ { - Sql: "select id, wo_lu_col, lu_col, lu_col = 5 from t2_wo_lookup where wo_lu_col = 2 and lu_col in (1, 2) for update", + Sql: "select id, wo_lu_col, nv_lu_col, lu_col, lu_col = 5 from t2_lookup where wo_lu_col = 2 and lu_col in (1, 2) for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "update t2_wo_lookup set lu_col = 5 where wo_lu_col = 2 and lu_col in (1, 2)", + Sql: "update t2_lookup set lu_col = 5 where wo_lu_col = 2 and lu_col in (1, 2)", BindVariables: map[string]*querypb.BindVariable{}, }} @@ -619,21 +619,21 @@ func TestUpdateUseHigherCostVindexIfBackfilling(t *testing.T) { assertQueries(t, sbc2, nil) } -func TestDeleteEqualWithWriteOnlyLookupUniqueVindex(t *testing.T) { +func TestDeleteEqualWithNoVerifyAndWriteOnlyLookupUniqueVindex(t *testing.T) { res := []*sqltypes.Result{sqltypes.MakeTestResult( - sqltypes.MakeTestFields("id|wo_lu_col|lu_col", "int64|int64|int64"), - "1|1|1", + sqltypes.MakeTestFields("id|wo_lu_col|nv_lu_col|lu_col", "int64|int64|int64|int64"), + "1|1|1|1", )} executor, sbc1, sbc2, sbcLookup := createCustomExecutorSetValues(executorVSchema, res) - _, err := executorExec(executor, "delete from t2_wo_lookup where wo_lu_col = 1", nil) + _, err := executorExec(executor, "delete from t2_lookup where wo_lu_col = 1", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{ { - Sql: "select id, wo_lu_col, lu_col from t2_wo_lookup where wo_lu_col = 1 for update", + Sql: "select id, wo_lu_col, nv_lu_col, lu_col from t2_lookup where wo_lu_col = 1 for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "delete from t2_wo_lookup where wo_lu_col = 1", + Sql: "delete from t2_lookup where wo_lu_col = 1", BindVariables: map[string]*querypb.BindVariable{}, }} @@ -645,13 +645,20 @@ func TestDeleteEqualWithWriteOnlyLookupUniqueVindex(t *testing.T) { }, } bq2 := &querypb.BoundQuery{ + Sql: "delete from nv_lu_idx where nv_lu_col = :nv_lu_col and keyspace_id = :keyspace_id", + BindVariables: map[string]*querypb.BindVariable{ + "keyspace_id": {Type: querypb.Type_VARBINARY, Value: []byte("\x16k@\xb4J\xbaK\xd6")}, + "nv_lu_col": sqltypes.Int64BindVariable(1), + }, + } + bq3 := &querypb.BoundQuery{ Sql: "delete from lu_idx where lu_col = :lu_col and keyspace_id = :keyspace_id", BindVariables: map[string]*querypb.BindVariable{ "keyspace_id": sqltypes.Uint64BindVariable(1), "lu_col": sqltypes.Int64BindVariable(1), }, } - lookWant := []*querypb.BoundQuery{bq1, bq2, bq1, bq2, bq1, bq2, bq1, bq2, bq1, bq2, bq1, bq2, bq1, bq2, bq1, bq2} + lookWant := []*querypb.BoundQuery{bq1, bq2, bq3, bq1, bq2, bq3, bq1, bq2, bq3, bq1, bq2, bq3, bq1, bq2, bq3, bq1, bq2, bq3, bq1, bq2, bq3, bq1, bq2, bq3} assertQueries(t, sbcLookup, lookWant) assertQueries(t, sbc1, wantQueries) assertQueries(t, sbc2, wantQueries) @@ -666,18 +673,18 @@ func TestDeleteEqualWithMultipleLookupVindex(t *testing.T) { )}) sbc1.SetResults([]*sqltypes.Result{sqltypes.MakeTestResult( - sqltypes.MakeTestFields("id|wo_lu_col|lu_col", "int64|int64|int64"), - "1|1|1", + sqltypes.MakeTestFields("id|wo_lu_col|nv_lu_col|lu_col", "int64|int64|int64|int64"), + "1|1|1|1", )}) - _, err := executorExec(executor, "delete from t2_wo_lookup where wo_lu_col = 1 and lu_col = 1", nil) + _, err := executorExec(executor, "delete from t2_lookup where wo_lu_col = 1 and lu_col = 1", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{ { - Sql: "select id, wo_lu_col, lu_col from t2_wo_lookup where wo_lu_col = 1 and lu_col = 1 for update", + Sql: "select id, wo_lu_col, nv_lu_col, lu_col from t2_lookup where wo_lu_col = 1 and lu_col = 1 for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "delete from t2_wo_lookup where wo_lu_col = 1 and lu_col = 1", + Sql: "delete from t2_lookup where wo_lu_col = 1 and lu_col = 1", BindVariables: map[string]*querypb.BindVariable{}, }} @@ -695,6 +702,12 @@ func TestDeleteEqualWithMultipleLookupVindex(t *testing.T) { "keyspace_id": {Type: querypb.Type_VARBINARY, Value: []byte("\x16k@\xb4J\xbaK\xd6")}, "wo_lu_col": sqltypes.Int64BindVariable(1), }, + }, { + Sql: "delete from nv_lu_idx where nv_lu_col = :nv_lu_col and keyspace_id = :keyspace_id", + BindVariables: map[string]*querypb.BindVariable{ + "keyspace_id": {Type: querypb.Type_VARBINARY, Value: []byte("\x16k@\xb4J\xbaK\xd6")}, + "nv_lu_col": sqltypes.Int64BindVariable(1), + }, }, { Sql: "delete from lu_idx where lu_col = :lu_col and keyspace_id = :keyspace_id", BindVariables: map[string]*querypb.BindVariable{ @@ -718,19 +731,19 @@ func TestDeleteUseHigherCostVindexIfBackfilling(t *testing.T) { )}) sbc1.SetResults([]*sqltypes.Result{sqltypes.MakeTestResult( - sqltypes.MakeTestFields("id|wo_lu_col|lu_col", "int64|int64|int64"), - "1|1|1", - "1|1|2", + sqltypes.MakeTestFields("id|wo_lu_col|nv_lu_col|lu_col", "int64|int64|int64|int64"), + "1|1|1|1", + "1|1|1|2", )}) - _, err := executorExec(executor, "delete from t2_wo_lookup where wo_lu_col = 1 and lu_col in (1, 2)", nil) + _, err := executorExec(executor, "delete from t2_lookup where wo_lu_col = 1 and lu_col in (1, 2)", nil) require.NoError(t, err) wantQueries := []*querypb.BoundQuery{ { - Sql: "select id, wo_lu_col, lu_col from t2_wo_lookup where wo_lu_col = 1 and lu_col in (1, 2) for update", + Sql: "select id, wo_lu_col, nv_lu_col, lu_col from t2_lookup where wo_lu_col = 1 and lu_col in (1, 2) for update", BindVariables: map[string]*querypb.BindVariable{}, }, { - Sql: "delete from t2_wo_lookup where wo_lu_col = 1 and lu_col in (1, 2)", + Sql: "delete from t2_lookup where wo_lu_col = 1 and lu_col in (1, 2)", BindVariables: map[string]*querypb.BindVariable{}, }} @@ -749,6 +762,12 @@ func TestDeleteUseHigherCostVindexIfBackfilling(t *testing.T) { "keyspace_id": {Type: querypb.Type_VARBINARY, Value: []byte("\x16k@\xb4J\xbaK\xd6")}, "wo_lu_col": sqltypes.Int64BindVariable(1), }, + }, { + Sql: "delete from nv_lu_idx where nv_lu_col = :nv_lu_col and keyspace_id = :keyspace_id", + BindVariables: map[string]*querypb.BindVariable{ + "keyspace_id": {Type: querypb.Type_VARBINARY, Value: []byte("\x16k@\xb4J\xbaK\xd6")}, + "nv_lu_col": sqltypes.Int64BindVariable(1), + }, }, { Sql: "delete from lu_idx where lu_col = :lu_col and keyspace_id = :keyspace_id", BindVariables: map[string]*querypb.BindVariable{ @@ -761,6 +780,12 @@ func TestDeleteUseHigherCostVindexIfBackfilling(t *testing.T) { "keyspace_id": {Type: querypb.Type_VARBINARY, Value: []byte("\x16k@\xb4J\xbaK\xd6")}, "wo_lu_col": sqltypes.Int64BindVariable(1), }, + }, { + Sql: "delete from nv_lu_idx where nv_lu_col = :nv_lu_col and keyspace_id = :keyspace_id", + BindVariables: map[string]*querypb.BindVariable{ + "keyspace_id": {Type: querypb.Type_VARBINARY, Value: []byte("\x16k@\xb4J\xbaK\xd6")}, + "nv_lu_col": sqltypes.Int64BindVariable(1), + }, }, { Sql: "delete from lu_idx where lu_col = :lu_col and keyspace_id = :keyspace_id", BindVariables: map[string]*querypb.BindVariable{ diff --git a/go/vt/vtgate/executor_framework_test.go b/go/vt/vtgate/executor_framework_test.go index 4a2a1e7cfec..d302a6451c7 100644 --- a/go/vt/vtgate/executor_framework_test.go +++ b/go/vt/vtgate/executor_framework_test.go @@ -119,9 +119,19 @@ var executorVSchema = ` "table": "TestUnsharded.wo_lu_idx", "from": "wo_lu_col", "to": "keyspace_id", - "write_only": "true" + "write_only": "true" }, - "owner": "t2_wo_lookup" + "owner": "t2_lookup" + }, + "t2_nv_lu_vdx": { + "type": "lookup_unique", + "params": { + "table": "TestUnsharded.nv_lu_idx", + "from": "nv_lu_col", + "to": "keyspace_id", + "no_verify": "true" + }, + "owner": "t2_lookup" }, "t2_lu_vdx": { "type": "lookup_hash_unique", @@ -130,7 +140,7 @@ var executorVSchema = ` "from": "lu_col", "to": "keyspace_id" }, - "owner": "t2_wo_lookup" + "owner": "t2_lookup" }, "regional_vdx": { "type": "region_experimental", @@ -294,7 +304,7 @@ var executorVSchema = ` } ] }, - "t2_wo_lookup": { + "t2_lookup": { "column_vindexes": [ { "column": "id", @@ -304,6 +314,10 @@ var executorVSchema = ` "column": "wo_lu_col", "name": "t2_wo_lu_vdx" }, + { + "column": "nv_lu_col", + "name": "t2_nv_lu_vdx" + }, { "column": "lu_col", "name": "t2_lu_vdx" @@ -350,6 +364,7 @@ var unshardedVSchema = ` } }, "wo_lu_idx": {}, + "nv_lu_idx": {}, "lu_idx": {}, "simple": {} } diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 6e80f3841aa..c52a878afa7 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -865,8 +865,9 @@ func TestExecutorShow(t *testing.T) { buildVarCharRow("TestExecutor", "name_user_map", "lookup_hash", "from=name; table=name_user_map; to=user_id", "user"), buildVarCharRow("TestExecutor", "regional_vdx", "region_experimental", "region_bytes=1", ""), buildVarCharRow("TestExecutor", "t1_lkp_vdx", "consistent_lookup_unique", "from=unq_col; table=t1_lkp_idx; to=keyspace_id", "t1"), - buildVarCharRow("TestExecutor", "t2_lu_vdx", "lookup_hash_unique", "from=lu_col; table=TestUnsharded.lu_idx; to=keyspace_id", "t2_wo_lookup"), - buildVarCharRow("TestExecutor", "t2_wo_lu_vdx", "lookup_unique", "from=wo_lu_col; table=TestUnsharded.wo_lu_idx; to=keyspace_id; write_only=true", "t2_wo_lookup"), + buildVarCharRow("TestExecutor", "t2_lu_vdx", "lookup_hash_unique", "from=lu_col; table=TestUnsharded.lu_idx; to=keyspace_id", "t2_lookup"), + buildVarCharRow("TestExecutor", "t2_nv_lu_vdx", "lookup_unique", "from=nv_lu_col; no_verify=true; table=TestUnsharded.nv_lu_idx; to=keyspace_id", "t2_lookup"), + buildVarCharRow("TestExecutor", "t2_wo_lu_vdx", "lookup_unique", "from=wo_lu_col; table=TestUnsharded.wo_lu_idx; to=keyspace_id; write_only=true", "t2_lookup"), buildVarCharRow("TestMultiCol", "multicol_vdx", "multicol", "column_bytes=1,3,4; column_count=3; column_vindex=hash,binary,unicode_loose_xxhash", ""), }, } @@ -1003,6 +1004,7 @@ func TestExecutorShow(t *testing.T) { buildVarCharRow("music_user_map"), buildVarCharRow("name_lastname_keyspace_id_map"), buildVarCharRow("name_user_map"), + buildVarCharRow("nv_lu_idx"), buildVarCharRow("simple"), buildVarCharRow("user_msgs"), buildVarCharRow("user_seq"), diff --git a/go/vt/vtgate/vindexes/lookup.go b/go/vt/vtgate/vindexes/lookup.go index c2577f17a1b..9ac514175df 100644 --- a/go/vt/vtgate/vindexes/lookup.go +++ b/go/vt/vtgate/vindexes/lookup.go @@ -46,6 +46,7 @@ func init() { type LookupNonUnique struct { name string writeOnly bool + noVerify bool lkp lookupInternal } @@ -136,7 +137,7 @@ func (ln *LookupNonUnique) MapResult(ids []sqltypes.Value, results []*sqltypes.R // Verify returns true if ids maps to ksids. func (ln *LookupNonUnique) Verify(ctx context.Context, vcursor VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) { - if ln.writeOnly { + if ln.writeOnly || ln.noVerify { out := make([]bool, len(ids)) for i := range ids { out[i] = true @@ -182,6 +183,7 @@ func (ln *LookupNonUnique) Query() (selQuery string, arguments []string) { // // autocommit: setting this to "true" will cause inserts to upsert and deletes to be ignored. // write_only: in this mode, Map functions return the full keyrange causing a full scatter. +// no_verify: in this mode, Verify will always succeed. func NewLookup(name string, m map[string]string) (Vindex, error) { lookup := &LookupNonUnique{name: name} @@ -194,6 +196,11 @@ func NewLookup(name string, m map[string]string) (Vindex, error) { return nil, err } + lookup.noVerify, err = boolFromMap(m, "no_verify") + if err != nil { + return nil, err + } + // if autocommit is on for non-unique lookup, upsert should also be on. upsert := cc.autocommit || cc.multiShardAutocommit if err := lookup.lkp.Init(m, cc.autocommit, upsert, cc.multiShardAutocommit); err != nil { @@ -218,6 +225,7 @@ func ksidsToValues(ksids [][]byte) []sqltypes.Value { type LookupUnique struct { name string writeOnly bool + noVerify bool lkp lookupInternal } @@ -256,6 +264,11 @@ func NewLookupUnique(name string, m map[string]string) (Vindex, error) { return nil, err } + lu.noVerify, err = boolFromMap(m, "no_verify") + if err != nil { + return nil, err + } + // Don't allow upserts for unique vindexes. if err := lu.lkp.Init(m, cc.autocommit, false /* upsert */, cc.multiShardAutocommit); err != nil { return nil, err @@ -320,7 +333,7 @@ func (lu *LookupUnique) MapResult(ids []sqltypes.Value, results []*sqltypes.Resu // Verify returns true if ids maps to ksids. func (lu *LookupUnique) Verify(ctx context.Context, vcursor VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) { - if lu.writeOnly { + if lu.writeOnly || lu.noVerify { out := make([]bool, len(ids)) for i := range ids { out[i] = true diff --git a/go/vt/vtgate/vindexes/lookup_hash.go b/go/vt/vtgate/vindexes/lookup_hash.go index ae838adf343..993b9655660 100644 --- a/go/vt/vtgate/vindexes/lookup_hash.go +++ b/go/vt/vtgate/vindexes/lookup_hash.go @@ -48,7 +48,7 @@ func init() { // LookupHash defines a vindex that uses a lookup table. // The table is expected to define the id column as unique. It's // NonUnique and a Lookup. -// Warning: This Vindex is being depcreated in favor of Lookup +// Warning: This Vindex is being deprecated in favor of Lookup type LookupHash struct { name string writeOnly bool diff --git a/go/vt/vtgate/vindexes/lookup_test.go b/go/vt/vtgate/vindexes/lookup_test.go index 538a009c9c5..df21f07c83d 100644 --- a/go/vt/vtgate/vindexes/lookup_test.go +++ b/go/vt/vtgate/vindexes/lookup_test.go @@ -281,6 +281,52 @@ func TestLookupNonUniqueVerify(t *testing.T) { utils.MustMatch(t, []bool{true, true}, got) } +func TestLookupNonUniqueNoVerify(t *testing.T) { + vindex, err := CreateVindex("lookup", "lookup", map[string]string{ + "table": "t", + "from": "fromc", + "to": "toc", + "no_verify": "true", + }) + require.NoError(t, err) + lookupNonUnique := vindex.(SingleColumn) + vc := &vcursor{numRows: 1} + + _, err = lookupNonUnique.Verify(context.Background(), vc, []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewInt64(2)}, [][]byte{[]byte("test1"), []byte("test2")}) + require.NoError(t, err) + + var wantqueries []*querypb.BoundQuery + utils.MustMatch(t, vc.queries, wantqueries) + + // Test query fail. + vc.mustFail = true + _, err = lookupNonUnique.Verify(context.Background(), vc, []sqltypes.Value{sqltypes.NewInt64(1)}, [][]byte{[]byte("\x16k@\xb4J\xbaK\xd6")}) + require.NoError(t, err) +} + +func TestLookupUniqueNoVerify(t *testing.T) { + vindex, err := CreateVindex("lookup_unique", "lookup_unique", map[string]string{ + "table": "t", + "from": "fromc", + "to": "toc", + "no_verify": "true", + }) + require.NoError(t, err) + lookupUnique := vindex.(SingleColumn) + vc := &vcursor{numRows: 1} + + _, err = lookupUnique.Verify(context.Background(), vc, []sqltypes.Value{sqltypes.NewInt64(1), sqltypes.NewInt64(2)}, [][]byte{[]byte("test1"), []byte("test2")}) + require.NoError(t, err) + + var wantqueries []*querypb.BoundQuery + utils.MustMatch(t, vc.queries, wantqueries) + + // Test query fail. + vc.mustFail = true + _, err = lookupUnique.Verify(context.Background(), vc, []sqltypes.Value{sqltypes.NewInt64(1)}, [][]byte{[]byte("\x16k@\xb4J\xbaK\xd6")}) + require.NoError(t, err) +} + func TestLookupNonUniqueVerifyAutocommit(t *testing.T) { vindex, err := CreateVindex("lookup", "lookup", map[string]string{ "table": "t",